def test_download_datasets(self): # test downloading a single file full_link_to_file = r'http://ibl.flatironinstitute.org/mainenlab/Subjects/clns0730'\ '/2018-08-24/1/licks.times.51852a2f-c76e-4c0c-95cb-9c7ba54be0f9.npy' file_name = wc.http_download_file(full_link_to_file, username=par.HTTP_DATA_SERVER_LOGIN, password=par.HTTP_DATA_SERVER_PWD) a = np.load(file_name) self.assertTrue(len(a) > 0) # test downloading a list of files links = [ r'http://ibl.flatironinstitute.org/mainenlab/Subjects/clns0730' '/2018-08-24/1/licks.times.51852a2f-c76e-4c0c-95cb-9c7ba54be0f9.npy', r'http://ibl.flatironinstitute.org/mainenlab/Subjects/clns0730' '/2018-08-24/1/probes.sitePositions.3ddd45be-7d24-4fc7-9dd3-a98717342af6.npy' ] file_list = wc.http_download_file_list( links, username=par.HTTP_DATA_SERVER_LOGIN, password=par.HTTP_DATA_SERVER_PWD) a = np.load(file_list[0]) b = np.load(file_list[1]) self.assertTrue(len(a) > 0) self.assertTrue(len(b) > 0)
def test_download_datasets_with_api(self): ac = self.ac # easier to debug in console test_data_uuid = self.test_data_uuid cache_dir = tempfile.mkdtemp() # Test 1: empty dir, dict mode dset = ac.get('/datasets/' + test_data_uuid) url = wc.dataset_record_to_url(dset) file_name = wc.http_download_file_list(url, username=par.HTTP_DATA_SERVER_LOGIN, password=par.HTTP_DATA_SERVER_PWD, verbose=True, cache_dir=cache_dir) # Test 2: empty dir, list mode dset = ac.get('/datasets?id=' + test_data_uuid) url = wc.dataset_record_to_url(dset) file_name = wc.http_download_file_list(url, username=par.HTTP_DATA_SERVER_LOGIN, password=par.HTTP_DATA_SERVER_PWD, verbose=True, cache_dir=cache_dir) self.assertTrue(os.path.isfile(file_name[0])) shutil.rmtree(cache_dir)
def download_raw_video(eid, cameras=None): """ Downloads the raw video from FlatIron or cache dir. This allows you to download just one of the three videos :param cameras: the specific camera to load (i.e. 'left', 'right', or 'body') If None all three videos are downloaded. :return: the file path(s) of the raw videos """ one = ONE() if cameras: cameras = [cameras] if isinstance(cameras, str) else cameras cam_files = ['_iblrig_{}Camera.raw.mp4'.format(cam) for cam in cameras] datasets = one._alyxClient.get('sessions/' + eid)['data_dataset_session_related'] urls = [ds['data_url'] for ds in datasets if ds['name'] in cam_files] cache_dir = one.path_from_eid(eid).joinpath('raw_video_data') if not os.path.exists(str(cache_dir)): os.mkdir(str(cache_dir)) else: # Check if file already downloaded # cam_files = [fi[:-4] for fi in cam_files] # Remove ext filenames = [ f for f in os.listdir(str(cache_dir)) if any([cam in f for cam in cam_files]) ] if filenames: return [cache_dir.joinpath(file) for file in filenames] http_download_file_list(urls, username=one._par.HTTP_DATA_SERVER_LOGIN, password=one._par.HTTP_DATA_SERVER_PWD, cache_dir=str(cache_dir)) return else: return one.load(eid, ['_iblrig_Camera.raw'], download_only=True)
def download_raw_video(self, cameras=None): """ Downloads the raw video from FlatIron or cache dir. This allows you to download just one of the three videos :param cameras: the specific camera to load (i.e. 'left', 'right', or 'body') If None all three videos are downloaded. :return: the file path(s) of the raw videos FIXME Currently returns if only one video already downloaded """ one = self.one eid = self._session_data['eid'] if cameras: cameras = [cameras] if isinstance(cameras, str) else cameras cam_files = [ '_iblrig_{}Camera.raw.mp4'.format(cam) for cam in cameras ] datasets = one.alyx.rest('sessions', 'read', id=eid)['data_dataset_session_related'] urls = [ ds['data_url'] for ds in datasets if ds['name'] in cam_files ] cache_dir = one.path_from_eid(eid).joinpath('raw_video_data') cache_dir.mkdir( exist_ok=True) # Create folder if doesn't already exist # Check if file already downloaded cam_files = [file[:-4] for file in cam_files] # Remove ext filenames = [ f for f in cache_dir.iterdir() if any([cam in str(f) for cam in cam_files]) ] if filenames: return [cache_dir.joinpath(file) for file in filenames] return http_download_file_list( urls, username=one._par.HTTP_DATA_SERVER_LOGIN, password=one._par.HTTP_DATA_SERVER_PWD, cache_dir=str(cache_dir)) else: return one.load(eid, ['_iblrig_Camera.raw'], download_only=True)