def GET(self, scope, name):
"""
List dataset replicas replicas.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
:returns: A dictionary containing all replicas information.
"""
header('Content-Type', 'application/x-json-stream')
deep = False
if ctx.query:
try:
params = loads(unquote(ctx.query[1:]))
except ValueError:
params = parse_qs(ctx.query[1:])
if 'deep' in params:
deep = params['deep'][0]
try:
for row in list_dataset_replicas(scope=scope, name=name,
deep=deep):
yield dumps(row, cls=APIEncoder) + '\n'
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
def get_datasets_at_rse(rse, prefix=None):
"""
:param rse: The RSE name
:param prefix: Character(s) to restrict the dataset search
:return: a dictionary with <dataset name>: <number of files>
"""
filters = {'scope': 'cms', 'did_type': DIDType.DATASET}
if prefix:
filters['name'] = '/' + prefix + '*'
with monitor.record_timer_block('cms_sync.time_rse_datasets'):
all_datasets = [
dataset['name']
for dataset in list_datasets_per_rse(rse=rse, filters=filters)
]
for dataset in all_datasets:
datasets = {
dataset: ds['available_length']
for ds in list_dataset_replicas(
scope='cms', name=dataset, deep=True)
if ds['rse'] == rse
}
# datasets = {dataset['name']: dataset['available_length']
# for dataset in list_datasets_per_rse(rse=rse, filters=filters, deep=True)}
return datasets
def get(self, scope, name):
"""
List dataset replicas.
.. :quickref: DatasetReplicas; List dataset replicas.
:query deep: Flag to ennable lookup at the file level.
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 401: Invalid auth token.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: A dictionary containing all replicas information.
"""
deep = request.args.get('deep', False)
try:
data = ""
for row in list_dataset_replicas(scope=scope, name=name,
deep=deep):
data += dumps(row, cls=APIEncoder) + '\n'
return Response(data, content_type='application/x-json-stream')
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
def get_datasets_at_rse(rse, prefix=None):
"""
:param rse: The RSE name
:param prefix: Character(s) to restrict the dataset search
:return: a dictionary with <dataset name>: <number of files>
"""
filters = {'scope': 'cms', 'did_type': DIDType.DATASET}
if prefix:
filters['name'] = '/' + prefix + '*'
account = SYNC_ACCOUNT_FMT % rse.lower()
rule_filters = {'account': account, 'scope': 'cms', 'did_type': DIDType.DATASET}
with monitor.record_timer_block('cms_sync.time_rse_datasets'):
synced_ds = {item['name'] for item in list_replication_rules(filters=rule_filters)
if item['expires_at'] is None and (prefix is None or item['name'].startswith('/' + prefix))}
all_datasets = [dataset['name'] for dataset in list_datasets_per_rse(rse=rse, filters=filters)]
logging.info('Getting all datasets at %s with prefix %s' % (rse, prefix))
datasets = {}
for dataset in all_datasets:
if dataset in synced_ds:
for ds in list_dataset_replicas(scope='cms', name=dataset, deep=True):
if ds['rse'] == rse:
datasets.update({dataset: ds['available_length']})
return datasets
def generate(_deep, vo):
for row in list_dataset_replicas(scope=scope, name=name, deep=_deep, vo=vo):
yield dumps(row, cls=APIEncoder) + '\n'
