def _make_dataclass_offline(self, eid, dataset_types=None, cache_dir=None, **kwargs):
if self._cache.size == 0:
return SessionDataInfo()
# select the session
npeid = parquet.str2np(eid)[0]
df = self._cache[self._cache['eid_0'] == npeid[0]]
df = df[df['eid_1'] == npeid[1]]
# select datasets
df = df[ismember(df['dataset_type'], dataset_types)[0]]
return SessionDataInfo.from_pandas(df, self._get_cache_dir(cache_dir))
def _make_dataclass(self, eid, dataset_types=None, cache_dir=None, dry_run=False,
clobber=False, offline=False, keep_uuid=False):
# if the input as an UUID, add the beginning of URL to it
cache_dir = self._get_cache_dir(cache_dir)
# get session json information as a dictionary from the alyx API
try:
ses = self.alyx.rest('sessions', 'read', id=eid)
except requests.HTTPError:
raise requests.HTTPError('Session ' + eid + ' does not exist')
# filter by dataset types
dc = SessionDataInfo.from_session_details(ses, dataset_types=dataset_types, eid=eid)
# loop over each dataset and download if necessary
with concurrent.futures.ThreadPoolExecutor(max_workers=NTHREADS) as executor:
futures = []
for ind in range(len(dc)):
if dc.url[ind] is None or dry_run:
futures.append(None)
else:
futures.append(executor.submit(
self.download_dataset, dc.url[ind], cache_dir=cache_dir, clobber=clobber,
offline=offline, keep_uuid=keep_uuid, file_size=dc.file_size[ind],
hash=dc.hash[ind]))
concurrent.futures.wait(list(filter(lambda x: x is not None, futures)))
for ind, future in enumerate(futures):
if future is None:
continue
dc.local_path[ind] = future.result()
# filter by daataset types and update the cache
self._update_cache(ses, dataset_types=dataset_types)
return dc
def _load(self, eid, dataset_types=None, dclass_output=False, dry_run=False, cache_dir=None,
download_only=False, clobber=False, offline=False, keep_uuid=False):
"""
From a Session ID and dataset types, queries Alyx database, downloads the data
from Globus, and loads into numpy array. Single session only
"""
# if the input as an UUID, add the beginning of URL to it
cache_dir = self._get_cache_dir(cache_dir)
if is_uuid_string(eid):
eid = '/sessions/' + eid
eid_str = eid[-36:]
# get session json information as a dictionary from the alyx API
try:
ses = self.alyx.get('/sessions/' + eid_str)
except requests.HTTPError:
raise requests.HTTPError('Session ' + eid_str + ' does not exist')
# ses = ses[0]
# if no dataset_type is provided:
# a) force the output to be a dictionary that provides context to the data
# b) download all types that have a data url specified whithin the alf folder
dataset_types = [dataset_types] if isinstance(dataset_types, str) else dataset_types
if not dataset_types or dataset_types == ['__all__']:
dclass_output = True
dc = SessionDataInfo.from_session_details(ses, dataset_types=dataset_types, eid=eid_str)
# loop over each dataset and download if necessary
for ind in range(len(dc)):
if dc.url[ind] and not dry_run:
relpath = PurePath(dc.url[ind].replace(self._par.HTTP_DATA_SERVER, '.')).parents[0]
cache_dir_file = PurePath(cache_dir, relpath)
Path(cache_dir_file).mkdir(parents=True, exist_ok=True)
dc.local_path[ind] = self._download_file(
dc.url[ind], str(cache_dir_file), clobber=clobber, offline=offline,
keep_uuid=keep_uuid, file_size=dc.file_size[ind], hash=dc.hash[ind])
# load the files content in variables if requested
if not download_only:
for ind, fil in enumerate(dc.local_path):
dc.data[ind] = load_file_content(fil)
# parse output arguments
if dclass_output:
return dc
# if required, parse the output as a list that matches dataset_types requested
list_out = []
for dt in dataset_types:
if dt not in dc.dataset_type:
_logger.warning('dataset ' + dt + ' not found for session: ' + eid_str)
list_out.append(None)
continue
for i, x, in enumerate(dc.dataset_type):
if dt == x:
if dc.data[i] is not None:
list_out.append(dc.data[i])
else:
list_out.append(dc.local_path[i])
return list_out
def setUp(self):
# Init connection to the database
dc1 = SessionDataInfo(
dataset_type='first dtype',
dataset_id='first uuid data',
local_path='/first/path',
eid='first uuid session',
url='first url',
data=np.array([1, 2, 3]),
)
dc2 = SessionDataInfo(
dataset_type='second dtype',
dataset_id='second uuid data',
local_path='/second/path',
eid='second uuid session',
url='second url',
data=np.array([1, 2, 3]),
)
self.dc1 = dc1
self.dc2 = dc2
self.dce = SessionDataInfo()
def _load(self,
eid,
dataset_types=None,
dclass_output=False,
dry_run=False,
cache_dir=None,
download_only=False,
clobber=False,
offline=False,
keep_uuid=False):
"""
From a Session ID and dataset types, queries Alyx database, downloads the data
from Globus, and loads into numpy array. Single session only
"""
# if the input as an UUID, add the beginning of URL to it
cache_dir = self._get_cache_dir(cache_dir)
if is_uuid_string(eid):
eid = '/sessions/' + eid
eid_str = eid[-36:]
# get session json information as a dictionary from the alyx API
try:
ses = self.alyx.get('/sessions/' + eid_str)
except requests.HTTPError:
raise requests.HTTPError('Session ' + eid_str + ' does not exist')
# ses = ses[0]
# if no dataset_type is provided:
# a) force the output to be a dictionary that provides context to the data
# b) download all types that have a data url specified whithin the alf folder
dataset_types = [dataset_types] if isinstance(dataset_types,
str) else dataset_types
if not dataset_types or dataset_types == ['__all__']:
dclass_output = True
# this performs the filtering
dc = SessionDataInfo.from_session_details(ses,
dataset_types=dataset_types,
eid=eid_str)
# loop over each dataset and download if necessary
with concurrent.futures.ThreadPoolExecutor(
max_workers=NTHREADS) as executor:
futures = []
for ind in range(len(dc)):
if dc.url[ind] is None or dry_run:
futures.append(None)
else:
futures.append(
executor.submit(self.download_dataset,
dc.url[ind],
cache_dir=cache_dir,
clobber=clobber,
offline=offline,
keep_uuid=keep_uuid,
file_size=dc.file_size[ind],
hash=dc.hash[ind]))
concurrent.futures.wait(
list(filter(lambda x: x is not None, futures)))
for ind, future in enumerate(futures):
if future is None:
continue
dc.local_path[ind] = future.result()
# load the files content in variables if requested
if not download_only:
for ind, fil in enumerate(dc.local_path):
dc.data[ind] = load_file_content(fil)
# parse output arguments
if dclass_output:
return dc
# if required, parse the output as a list that matches dataset_types requested
list_out = []
for dt in dataset_types:
if dt not in dc.dataset_type:
_logger.warning('dataset ' + dt + ' not found for session: ' +
eid_str)
list_out.append(None)
continue
for i, x, in enumerate(dc.dataset_type):
if dt == x:
if dc.data[i] is not None:
list_out.append(dc.data[i])
else:
list_out.append(dc.local_path[i])
return list_out
def load(self,
eid,
dataset_types=None,
dclass_output=False,
dry_run=False,
cache_dir=None,
download_only=False,
clobber=False):
"""
From a Session ID and dataset types, queries Alyx database, downloads the data
from Globus, and loads into numpy array.
:param eid: Experiment ID, for IBL this is the UUID of the Session as per Alyx
database. Could be a full Alyx URL:
'http://localhost:8000/sessions/698361f6-b7d0-447d-a25d-42afdef7a0da' or only the UUID:
'698361f6-b7d0-447d-a25d-42afdef7a0da'
:type eid: str
:param dataset_types: [None]: Alyx dataset types to be returned.
:type dataset_types: list
:param dclass_output: [False]: forces the output as dataclass to provide context.
:type dclass_output: bool
If None or an empty dataset_type is specified, the output will be a dictionary by default.
:param cache_dir: temporarly overrides the cache_dir from the parameter file
:type cache_dir: str
:param download_only: do not attempt to load data in memory, just download the files
:type download_only: bool
:param clobber: force downloading even if files exists locally
:type clobber: bool
:return: List of numpy arrays matching the size of dataset_types parameter, OR
a dataclass containing arrays and context data.
:rtype: list, dict, dataclass SessionDataInfo
"""
# if the input as an UUID, add the beginning of URL to it
cache_dir = self._get_cache_dir(cache_dir)
if is_uuid_string(eid):
eid = '/sessions/' + eid
eid_str = eid[-36:]
# get session json information as a dictionary from the alyx API
ses = self.alyx.get('/sessions?id=' + eid_str)
if not ses:
raise FileNotFoundError('Session ' + eid_str + ' does not exist')
ses = ses[0]
# if no dataset_type is provided:
# a) force the output to be a dictionary that provides context to the data
# b) download all types that have a data url specified
dataset_types = [dataset_types] if isinstance(dataset_types,
str) else dataset_types
if not dataset_types:
dclass_output = True
dataset_types = [
d['dataset_type'] for d in ses['data_dataset_session_related']
if d['data_url']
]
dc = SessionDataInfo.from_session_details(ses,
dataset_types=dataset_types)
# loop over each dataset and download if necessary
for ind in range(len(dc)):
if dc.url[ind] and not dry_run:
relpath = PurePath(dc.url[ind].replace(
self._par.HTTP_DATA_SERVER, '.')).parents[0]
cache_dir_file = PurePath(cache_dir, relpath)
Path(cache_dir_file).mkdir(parents=True, exist_ok=True)
dc.local_path[ind] = self._download_file(
dc.url[ind], str(cache_dir_file), clobber)
# load the files content in variables if requested
if not download_only:
for ind, fil in enumerate(dc.local_path):
dc.data[ind] = _load_file_content(fil)
# parse output arguments
if dclass_output:
return dc
# if required, parse the output as a list that matches dataset_types requested
list_out = []
for dt in dataset_types:
if dt not in dc.dataset_type:
list_out.append(None)
continue
for i, x, in enumerate(dc.dataset_type):
if dt == x:
list_out.append(dc.data[i])
return list_out