def fetch_file_replicas(self, scope, name): destination_rse = self.rucio.instance_config.get('destination_rse') replicas = [] rucio_replicas = self.rucio.get_replicas(scope, name) def rucio_replica_mapper(rucio_replica, _): rses = rucio_replica['rses'] states = rucio_replica['states'] scope = rucio_replica['scope'] name = rucio_replica['name'] size = rucio_replica['bytes'] pfn = None if (destination_rse in rses) and (destination_rse in states): if states[destination_rse] == 'AVAILABLE': pfns = rses[destination_rse] if len(pfns) > 0: pfn = pfns[0] did = scope + ':' + name return PfnFileReplica(pfn=pfn, did=did, size=size) replicas = utils.map(rucio_replicas, rucio_replica_mapper) return replicas
def get_did_details(self, scope, name, force_fetch=False): attached_file_replicas = self.get_attached_file_replicas( scope, name, force_fetch) complete = utils.find(lambda x: x.path is None, attached_file_replicas) is None status = ReplicaModeHandler.STATUS_NOT_AVAILABLE if not complete: status = self.get_did_status(scope, name) def result_mapper(file_replica, _): file_did = file_replica.did path = file_replica.path size = file_replica.size if path is None: # This is to handle newly-attached files in which the replication rule hasn't been reevaluated by the judger daemon. result_status = status if status != ReplicaModeHandler.STATUS_OK else ReplicaModeHandler.STATUS_REPLICATING return dict(status=result_status, did=file_did, path=None, size=size) return dict(status=ReplicaModeHandler.STATUS_OK, did=file_did, path=path, size=size) results = utils.map(attached_file_replicas, result_mapper) return results
def test_map(): iterable = [1, 2, 3, 4, 5, 6, 7, 8] expected_result = [2, 3, 4, 5, 6, 7, 8, 9] assert utils.map( iterable, lambda x, _: x + 1) == expected_result, "Invalid return value"
def get_did_details(self, scope, name, force_fetch=False): did = scope + ':' + name attached_files = self._get_attached_files(scope, name, force_fetch) dest_dir_exists = self._dest_dir_exists(did) is_downloading = self._is_downloading(did) paths = self._get_file_paths(did) def result_mapper(file, _): if not dest_dir_exists: return dict(status=DownloadModeHandler.STATUS_NOT_AVAILABLE, did=file.did, path=None, size=file.size) if is_downloading: return dict(status=DownloadModeHandler.STATUS_REPLICATING, did=file.did, path=None, size=file.size) if not paths: return dict(status=DownloadModeHandler.STATUS_STUCK, did=file.did, path=None, size=file.size) path = paths.get(file.did) if not os.path.isfile(path): return dict(status=DownloadModeHandler.STATUS_STUCK, did=file.did, path=None, size=file.size) return dict(status=DownloadModeHandler.STATUS_OK, did=file.did, path=path, size=file.size) results = utils.map(attached_files, result_mapper) return results
def _get_attached_files(self, scope, name, force_fetch=False): did = scope + ':' + name attached_files = self.db.get_attached_files(self.namespace, did) if not force_fetch else None if not attached_files: rucio_attached_files = self.rucio.get_files(scope, name) def mapper(d, _): return AttachedFile(did=(d.get('scope') + ':' + d.get('name')), size=d.get('bytes')) attached_files = utils.map(rucio_attached_files, mapper) return attached_files
def test_list_contents__should_ignore_dotfiles(mocker): def mock_listdir(_): return ['.abc', '.def', 'ghi'] def mock_isfile(_): return True mocker.patch.object(os, 'listdir', side_effect=mock_listdir) mocker.patch.object(os.path, 'isfile', side_effect=mock_isfile) mocker.patch('rucio_jupyterlab.handlers.file_browser.os', os) result = FileBrowserHandlerImpl.list_contents('') file_names = utils.map(result, lambda x, _: x['name']) assert file_names == ['ghi'], "Not removing dotfiles"
def search_did(self, scope, name, search_type, limit): wildcard_enabled = self.rucio.instance_config.get('wildcard_enabled', False) if ('*' in name or '%' in name) and not wildcard_enabled: raise WildcardDisallowedException() dids = self.rucio.search_did(scope, name, search_type, limit) def mapper(entry, _): return { 'did': entry.get('scope') + ':' + entry.get('name'), 'size': entry.get('bytes'), 'type': entry.get('did_type').lower() } result = utils.map(dids, mapper) return result
def test_list_contents__is_file_false__should_type_dir(mocker): def mock_listdir(_): return ['ghi', 'jkl'] def mock_isfile(_): return False def mock_isdir(_): return True mocker.patch.object(os, 'listdir', side_effect=mock_listdir) mocker.patch.object(os.path, 'isfile', side_effect=mock_isfile) mocker.patch.object(os.path, 'isdir', side_effect=mock_isdir) mocker.patch('rucio_jupyterlab.handlers.file_browser.os', os) result = FileBrowserHandlerImpl.list_contents('') types = utils.map(result, lambda x, _: x['type']) assert types == ['dir', 'dir'], "Not removing dotfiles"
def get_attached_file_replicas(self, scope, name, force_fetch=False): did = scope + ':' + name attached_files = self.db.get_attached_files( self.namespace, did) if not force_fetch else None pfn_file_replicas = self.get_all_pfn_file_replicas_from_db( attached_files) if attached_files else None if pfn_file_replicas is None: pfn_file_replicas = self.fetch_attached_pfn_file_replicas( scope, name) def file_replica_mapper(pfn_file_replica, _): file_did = pfn_file_replica.did pfn = pfn_file_replica.pfn size = pfn_file_replica.size path = self.translate_pfn_to_path(pfn) if pfn else None return FileReplica(did=file_did, path=path, size=size) file_replicas = utils.map(pfn_file_replicas, file_replica_mapper) return file_replicas
def list_contents(path): home = os.path.expanduser('~') path = os.path.expanduser(path) if not path: path = home if path[0] != '/' and path[0] != '~': path = os.path.join(home, path) if not os.path.exists(path): return None items = os.listdir(path) def items_mapper(item, _): full_path = os.path.join(path, item) item_type = 'file' if os.path.isfile(full_path) else 'dir' return dict(type=item_type, name=item, path=full_path) items = utils.map(items, items_mapper) items = utils.filter(items, lambda x, _: x['name'][0] != '.') return items