def fetch_file_replicas(self, scope, name):
destination_rse = self.rucio.instance_config.get('destination_rse')
replicas = []
rucio_replicas = self.rucio.get_replicas(scope, name)
def rucio_replica_mapper(rucio_replica, _):
rses = rucio_replica['rses']
states = rucio_replica['states']
scope = rucio_replica['scope']
name = rucio_replica['name']
size = rucio_replica['bytes']
pfn = None
if (destination_rse in rses) and (destination_rse in states):
if states[destination_rse] == 'AVAILABLE':
pfns = rses[destination_rse]
if len(pfns) > 0:
pfn = pfns[0]
did = scope + ':' + name
return PfnFileReplica(pfn=pfn, did=did, size=size)
replicas = utils.map(rucio_replicas, rucio_replica_mapper)
return replicas
def get_did_details(self, scope, name, force_fetch=False):
attached_file_replicas = self.get_attached_file_replicas(
scope, name, force_fetch)
complete = utils.find(lambda x: x.path is None,
attached_file_replicas) is None
status = ReplicaModeHandler.STATUS_NOT_AVAILABLE
if not complete:
status = self.get_did_status(scope, name)
def result_mapper(file_replica, _):
file_did = file_replica.did
path = file_replica.path
size = file_replica.size
if path is None:
# This is to handle newly-attached files in which the replication rule hasn't been reevaluated by the judger daemon.
result_status = status if status != ReplicaModeHandler.STATUS_OK else ReplicaModeHandler.STATUS_REPLICATING
return dict(status=result_status,
did=file_did,
path=None,
size=size)
return dict(status=ReplicaModeHandler.STATUS_OK,
did=file_did,
path=path,
size=size)
results = utils.map(attached_file_replicas, result_mapper)
return results
def test_map():
iterable = [1, 2, 3, 4, 5, 6, 7, 8]
expected_result = [2, 3, 4, 5, 6, 7, 8, 9]
assert utils.map(
iterable,
lambda x, _: x + 1) == expected_result, "Invalid return value"
def get_did_details(self, scope, name, force_fetch=False):
did = scope + ':' + name
attached_files = self._get_attached_files(scope, name, force_fetch)
dest_dir_exists = self._dest_dir_exists(did)
is_downloading = self._is_downloading(did)
paths = self._get_file_paths(did)
def result_mapper(file, _):
if not dest_dir_exists:
return dict(status=DownloadModeHandler.STATUS_NOT_AVAILABLE, did=file.did, path=None, size=file.size)
if is_downloading:
return dict(status=DownloadModeHandler.STATUS_REPLICATING, did=file.did, path=None, size=file.size)
if not paths:
return dict(status=DownloadModeHandler.STATUS_STUCK, did=file.did, path=None, size=file.size)
path = paths.get(file.did)
if not os.path.isfile(path):
return dict(status=DownloadModeHandler.STATUS_STUCK, did=file.did, path=None, size=file.size)
return dict(status=DownloadModeHandler.STATUS_OK, did=file.did, path=path, size=file.size)
results = utils.map(attached_files, result_mapper)
return results
def _get_attached_files(self, scope, name, force_fetch=False):
did = scope + ':' + name
attached_files = self.db.get_attached_files(self.namespace, did) if not force_fetch else None
if not attached_files:
rucio_attached_files = self.rucio.get_files(scope, name)
def mapper(d, _):
return AttachedFile(did=(d.get('scope') + ':' + d.get('name')), size=d.get('bytes'))
attached_files = utils.map(rucio_attached_files, mapper)
return attached_files
def test_list_contents__should_ignore_dotfiles(mocker):
def mock_listdir(_):
return ['.abc', '.def', 'ghi']
def mock_isfile(_):
return True
mocker.patch.object(os, 'listdir', side_effect=mock_listdir)
mocker.patch.object(os.path, 'isfile', side_effect=mock_isfile)
mocker.patch('rucio_jupyterlab.handlers.file_browser.os', os)
result = FileBrowserHandlerImpl.list_contents('')
file_names = utils.map(result, lambda x, _: x['name'])
assert file_names == ['ghi'], "Not removing dotfiles"
def search_did(self, scope, name, search_type, limit):
wildcard_enabled = self.rucio.instance_config.get('wildcard_enabled', False)
if ('*' in name or '%' in name) and not wildcard_enabled:
raise WildcardDisallowedException()
dids = self.rucio.search_did(scope, name, search_type, limit)
def mapper(entry, _):
return {
'did': entry.get('scope') + ':' + entry.get('name'),
'size': entry.get('bytes'),
'type': entry.get('did_type').lower()
}
result = utils.map(dids, mapper)
return result
def test_list_contents__is_file_false__should_type_dir(mocker):
def mock_listdir(_):
return ['ghi', 'jkl']
def mock_isfile(_):
return False
def mock_isdir(_):
return True
mocker.patch.object(os, 'listdir', side_effect=mock_listdir)
mocker.patch.object(os.path, 'isfile', side_effect=mock_isfile)
mocker.patch.object(os.path, 'isdir', side_effect=mock_isdir)
mocker.patch('rucio_jupyterlab.handlers.file_browser.os', os)
result = FileBrowserHandlerImpl.list_contents('')
types = utils.map(result, lambda x, _: x['type'])
assert types == ['dir', 'dir'], "Not removing dotfiles"
def get_attached_file_replicas(self, scope, name, force_fetch=False):
did = scope + ':' + name
attached_files = self.db.get_attached_files(
self.namespace, did) if not force_fetch else None
pfn_file_replicas = self.get_all_pfn_file_replicas_from_db(
attached_files) if attached_files else None
if pfn_file_replicas is None:
pfn_file_replicas = self.fetch_attached_pfn_file_replicas(
scope, name)
def file_replica_mapper(pfn_file_replica, _):
file_did = pfn_file_replica.did
pfn = pfn_file_replica.pfn
size = pfn_file_replica.size
path = self.translate_pfn_to_path(pfn) if pfn else None
return FileReplica(did=file_did, path=path, size=size)
file_replicas = utils.map(pfn_file_replicas, file_replica_mapper)
return file_replicas
def list_contents(path):
home = os.path.expanduser('~')
path = os.path.expanduser(path)
if not path:
path = home
if path[0] != '/' and path[0] != '~':
path = os.path.join(home, path)
if not os.path.exists(path):
return None
items = os.listdir(path)
def items_mapper(item, _):
full_path = os.path.join(path, item)
item_type = 'file' if os.path.isfile(full_path) else 'dir'
return dict(type=item_type, name=item, path=full_path)
items = utils.map(items, items_mapper)
items = utils.filter(items, lambda x, _: x['name'][0] != '.')
return items
