def test_download_datasets(self):
# test downloading a single file
full_link_to_file = r'http://ibl.flatironinstitute.org/mainenlab/Subjects/clns0730'\
'/2018-08-24/1/licks.times.51852a2f-c76e-4c0c-95cb-9c7ba54be0f9.npy'
file_name = wc.http_download_file(full_link_to_file,
username=par.HTTP_DATA_SERVER_LOGIN,
password=par.HTTP_DATA_SERVER_PWD)
a = np.load(file_name)
self.assertTrue(len(a) > 0)
# test downloading a list of files
links = [
r'http://ibl.flatironinstitute.org/mainenlab/Subjects/clns0730'
'/2018-08-24/1/licks.times.51852a2f-c76e-4c0c-95cb-9c7ba54be0f9.npy',
r'http://ibl.flatironinstitute.org/mainenlab/Subjects/clns0730'
'/2018-08-24/1/probes.sitePositions.3ddd45be-7d24-4fc7-9dd3-a98717342af6.npy'
]
file_list = wc.http_download_file_list(
links,
username=par.HTTP_DATA_SERVER_LOGIN,
password=par.HTTP_DATA_SERVER_PWD)
a = np.load(file_list[0])
b = np.load(file_list[1])
self.assertTrue(len(a) > 0)
self.assertTrue(len(b) > 0)
def test_download_datasets_with_api(self):
ac = self.ac # easier to debug in console
test_data_uuid = self.test_data_uuid
cache_dir = tempfile.mkdtemp()
# Test 1: empty dir, dict mode
dset = ac.get('/datasets/' + test_data_uuid)
url = wc.dataset_record_to_url(dset)
file_name = wc.http_download_file_list(url, username=par.HTTP_DATA_SERVER_LOGIN,
password=par.HTTP_DATA_SERVER_PWD,
verbose=True, cache_dir=cache_dir)
# Test 2: empty dir, list mode
dset = ac.get('/datasets?id=' + test_data_uuid)
url = wc.dataset_record_to_url(dset)
file_name = wc.http_download_file_list(url, username=par.HTTP_DATA_SERVER_LOGIN,
password=par.HTTP_DATA_SERVER_PWD,
verbose=True, cache_dir=cache_dir)
self.assertTrue(os.path.isfile(file_name[0]))
shutil.rmtree(cache_dir)
def download_raw_video(eid, cameras=None):
"""
Downloads the raw video from FlatIron or cache dir.
This allows you to download just one of the
three videos
:param cameras: the specific camera to load
(i.e. 'left', 'right', or 'body') If None all
three videos are downloaded.
:return: the file path(s) of the raw videos
"""
one = ONE()
if cameras:
cameras = [cameras] if isinstance(cameras, str) else cameras
cam_files = ['_iblrig_{}Camera.raw.mp4'.format(cam) for cam in cameras]
datasets = one._alyxClient.get('sessions/' +
eid)['data_dataset_session_related']
urls = [ds['data_url'] for ds in datasets if ds['name'] in cam_files]
cache_dir = one.path_from_eid(eid).joinpath('raw_video_data')
if not os.path.exists(str(cache_dir)):
os.mkdir(str(cache_dir))
else: # Check if file already downloaded
# cam_files = [fi[:-4] for fi in cam_files] # Remove ext
filenames = [
f for f in os.listdir(str(cache_dir))
if any([cam in f for cam in cam_files])
]
if filenames:
return [cache_dir.joinpath(file) for file in filenames]
http_download_file_list(urls,
username=one._par.HTTP_DATA_SERVER_LOGIN,
password=one._par.HTTP_DATA_SERVER_PWD,
cache_dir=str(cache_dir))
return
else:
return one.load(eid, ['_iblrig_Camera.raw'], download_only=True)
def download_raw_video(self, cameras=None):
"""
Downloads the raw video from FlatIron or cache dir. This allows you to download just one
of the three videos
:param cameras: the specific camera to load (i.e. 'left', 'right', or 'body') If None all
three videos are downloaded.
:return: the file path(s) of the raw videos
FIXME Currently returns if only one video already downloaded
"""
one = self.one
eid = self._session_data['eid']
if cameras:
cameras = [cameras] if isinstance(cameras, str) else cameras
cam_files = [
'_iblrig_{}Camera.raw.mp4'.format(cam) for cam in cameras
]
datasets = one.alyx.rest('sessions', 'read',
id=eid)['data_dataset_session_related']
urls = [
ds['data_url'] for ds in datasets if ds['name'] in cam_files
]
cache_dir = one.path_from_eid(eid).joinpath('raw_video_data')
cache_dir.mkdir(
exist_ok=True) # Create folder if doesn't already exist
# Check if file already downloaded
cam_files = [file[:-4] for file in cam_files] # Remove ext
filenames = [
f for f in cache_dir.iterdir()
if any([cam in str(f) for cam in cam_files])
]
if filenames:
return [cache_dir.joinpath(file) for file in filenames]
return http_download_file_list(
urls,
username=one._par.HTTP_DATA_SERVER_LOGIN,
password=one._par.HTTP_DATA_SERVER_PWD,
cache_dir=str(cache_dir))
else:
return one.load(eid, ['_iblrig_Camera.raw'], download_only=True)
