def GET(self, scope, name): """ List dataset replicas replicas. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError :returns: A dictionary containing all replicas information. """ header('Content-Type', 'application/x-json-stream') deep = False if ctx.query: try: params = loads(unquote(ctx.query[1:])) except ValueError: params = parse_qs(ctx.query[1:]) if 'deep' in params: deep = params['deep'][0] try: for row in list_dataset_replicas(scope=scope, name=name, deep=deep): yield dumps(row, cls=APIEncoder) + '\n' except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error)
def get_datasets_at_rse(rse, prefix=None): """ :param rse: The RSE name :param prefix: Character(s) to restrict the dataset search :return: a dictionary with <dataset name>: <number of files> """ filters = {'scope': 'cms', 'did_type': DIDType.DATASET} if prefix: filters['name'] = '/' + prefix + '*' with monitor.record_timer_block('cms_sync.time_rse_datasets'): all_datasets = [ dataset['name'] for dataset in list_datasets_per_rse(rse=rse, filters=filters) ] for dataset in all_datasets: datasets = { dataset: ds['available_length'] for ds in list_dataset_replicas( scope='cms', name=dataset, deep=True) if ds['rse'] == rse } # datasets = {dataset['name']: dataset['available_length'] # for dataset in list_datasets_per_rse(rse=rse, filters=filters, deep=True)} return datasets
def get(self, scope, name): """ List dataset replicas. .. :quickref: DatasetReplicas; List dataset replicas. :query deep: Flag to ennable lookup at the file level. :resheader Content-Type: application/x-json-stream :status 200: OK. :status 401: Invalid auth token. :status 406: Not Acceptable. :status 500: Internal Error. :returns: A dictionary containing all replicas information. """ deep = request.args.get('deep', False) try: data = "" for row in list_dataset_replicas(scope=scope, name=name, deep=deep): data += dumps(row, cls=APIEncoder) + '\n' return Response(data, content_type='application/x-json-stream') except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500
def get_datasets_at_rse(rse, prefix=None): """ :param rse: The RSE name :param prefix: Character(s) to restrict the dataset search :return: a dictionary with <dataset name>: <number of files> """ filters = {'scope': 'cms', 'did_type': DIDType.DATASET} if prefix: filters['name'] = '/' + prefix + '*' account = SYNC_ACCOUNT_FMT % rse.lower() rule_filters = {'account': account, 'scope': 'cms', 'did_type': DIDType.DATASET} with monitor.record_timer_block('cms_sync.time_rse_datasets'): synced_ds = {item['name'] for item in list_replication_rules(filters=rule_filters) if item['expires_at'] is None and (prefix is None or item['name'].startswith('/' + prefix))} all_datasets = [dataset['name'] for dataset in list_datasets_per_rse(rse=rse, filters=filters)] logging.info('Getting all datasets at %s with prefix %s' % (rse, prefix)) datasets = {} for dataset in all_datasets: if dataset in synced_ds: for ds in list_dataset_replicas(scope='cms', name=dataset, deep=True): if ds['rse'] == rse: datasets.update({dataset: ds['available_length']}) return datasets
def generate(_deep, vo): for row in list_dataset_replicas(scope=scope, name=name, deep=_deep, vo=vo): yield dumps(row, cls=APIEncoder) + '\n'