def GET(self, account, name):
"""
Return all rules of a given subscription id.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 Internal Error
:param scope: The scope name.
"""
header('Content-Type', 'application/x-json-stream')
state = None
if ctx.query:
params = parse_qs(ctx.query[1:])
if 'state' in params:
state = params['state'][0]
try:
subscriptions = [subscription['id'] for subscription in list_subscriptions(name=name, account=account)]
if len(subscriptions) > 0:
if state:
for rule in list_replication_rules({'subscription_id': subscriptions[0], 'state': state}):
yield dumps(rule, cls=APIEncoder) + '\n'
else:
for rule in list_replication_rules({'subscription_id': subscriptions[0]}):
yield dumps(rule, cls=APIEncoder) + '\n'
except RuleNotFound, error:
raise generate_http_error(404, 'RuleNotFound', error.args[0][0])
def generate(vo):
if len(subscriptions) > 0:
if state:
for rule in list_replication_rules({'subscription_id': subscriptions[0], 'state': state}, vo=vo):
yield render_json(**rule) + '\n'
else:
for rule in list_replication_rules({'subscription_id': subscriptions[0]}, vo=vo):
yield render_json(**rule) + '\n'
def get(self, account, name):
"""
Return all rules of a given subscription id.
.. :quickref: Rules; Get subscription rules.
:param account: The account name.
:param name: The subscription name.
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: Rule Not Found.
:status 404: Subscription Not Found.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: Line separated list of dictionaries with rule information.
"""
state = request.args.get('state', None)
try:
subscriptions = [
subscription['id']
for subscription in list_subscriptions(name=name,
account=account)
]
data = ""
if len(subscriptions) > 0:
if state:
for rule in list_replication_rules({
'subscription_id':
subscriptions[0],
'state':
state
}):
data += dumps(rule, cls=APIEncoder) + '\n'
else:
for rule in list_replication_rules(
{'subscription_id': subscriptions[0]}):
data += dumps(rule, cls=APIEncoder) + '\n'
return Response(data, content_type='application/x-json-stream')
except RuleNotFound as error:
return generate_http_error_flask(404, 'RuleNotFound',
error.args[0])
except SubscriptionNotFound as error:
return generate_http_error_flask(404, 'SubscriptionNotFound',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
return error, 500
def GET(self, account, name):
"""
Return all rules of a given subscription id.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Not Found
:param scope: The scope name.
"""
header('Content-Type', 'application/x-json-stream')
state = None
if ctx.query:
params = parse_qs(ctx.query[1:])
if 'state' in params:
state = params['state'][0]
try:
subscriptions = [subscription['id'] for subscription in list_subscriptions(name=name, account=account)]
if len(subscriptions) > 0:
if state == 'OK':
state = RuleState.OK
if state == 'Replicating':
state = RuleState.REPLICATING
if state == 'Stuck':
state = RuleState.STUCK
for rule in list_replication_rules({'subscription_id': subscriptions[0], 'state': state}):
yield dumps(rule, cls=APIEncoder) + '\n'
except RuleNotFound, e:
raise generate_http_error(404, 'RuleNotFound', e.args[0][0])
def GET(self):
"""
Return all rules of a given account.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Not Found
:param scope: The scope name.
"""
header('Content-Type', 'application/x-json-stream')
filters = {}
if ctx.query:
params = dict(parse_qsl(ctx.query[1:]))
filters.update(params)
try:
for rule in list_replication_rules(filters=filters):
yield dumps(rule, cls=APIEncoder) + '\n'
except RuleNotFound as error:
raise generate_http_error(404, 'RuleNotFound', error.args[0])
except Exception as error:
print format_exc()
raise InternalError(error)
def generate(vo):
for rule in list_replication_rules(
{
'scope': scope,
'name': name
}, vo=vo):
yield dumps(rule, cls=APIEncoder) + '\n'
def GET(self, scope, name):
"""
Return all rules of a given DID.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
406 Not Acceptable
404 Not Found
:param scope: The scope name.
"""
header('Content-Type', 'application/x-json-stream')
try:
for rule in list_replication_rules({
'scope': scope,
'name': name
},
vo=ctx.env.get('vo')):
yield dumps(rule, cls=APIEncoder) + '\n'
except RuleNotFound as error:
raise generate_http_error(404, 'RuleNotFound', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
raise InternalError(error)
def get(self):
"""
Return all rules of a given account.
.. :quickref: AllRule; get all rules for account
:resheader Content-Type: application/x-json-stream
:status 200: Rule found
:status 401: Invalid Auth Token
:status 404: no rule found for id
:status 406: Not Acceptable
:query scope: The scope name.
"""
filters = {}
if request.args:
params = dict(request.args)
filters.update(params)
try:
data = ""
for rule in list_replication_rules(filters=filters):
data += dumps(rule, cls=APIEncoder) + '\n'
return Response(data, content_type="application/x-json-stream")
except RuleNotFound as error:
return generate_http_error_flask(404, 'RuleNotFound',
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
def get(self, scope, name):
"""
Return all rules of a given DID.
.. :quickref: Rules; List rules of DID.
:resheader Content-Type: application/x-json-stream
:param scope: The scope of the data identifier.
:param name: The name of the data identifier.
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: DID not found
:status 500: Database Exception
:returns: List of replication rules.
"""
try:
data = ""
for rule in list_replication_rules({'scope': scope, 'name': name}):
data += dumps(rule, cls=APIEncoder) + '\n'
return Response(data, content_type="application/x-json-stream")
except RuleNotFound as error:
return generate_http_error_flask(404, 'RuleNotFound',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
return error, 500
def get(self, account):
"""
Return all rules of a given account.
.. :quickref: Rules; Get rules for account.
:param scope: The scope name.
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 401: Invalid auth token.
:status 404: Rule not found.
:status 406: Not Acceptable.
:status 500: Database exception.
:returns: Line separated list of rules.
"""
filters = {'account': account}
for k, v in request.args.items():
filters[k] = v
try:
data = ""
for rule in list_replication_rules(filters=filters):
data += dumps(rule, cls=APIEncoder) + '\n'
return Response(data, content_type="application/x-json-stream")
except RuleNotFound as error:
return generate_http_error_flask(404, 'RuleNotFound',
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
def get_datasets_at_rse(rse, prefix=None):
"""
:param rse: The RSE name
:param prefix: Character(s) to restrict the dataset search
:return: a dictionary with <dataset name>: <number of files>
"""
filters = {'scope': 'cms', 'did_type': DIDType.DATASET}
if prefix:
filters['name'] = '/' + prefix + '*'
account = SYNC_ACCOUNT_FMT % rse.lower()
rule_filters = {'account': account, 'scope': 'cms', 'did_type': DIDType.DATASET}
with monitor.record_timer_block('cms_sync.time_rse_datasets'):
synced_ds = {item['name'] for item in list_replication_rules(filters=rule_filters)
if item['expires_at'] is None and (prefix is None or item['name'].startswith('/' + prefix))}
all_datasets = [dataset['name'] for dataset in list_datasets_per_rse(rse=rse, filters=filters)]
logging.info('Getting all datasets at %s with prefix %s' % (rse, prefix))
datasets = {}
for dataset in all_datasets:
if dataset in synced_ds:
for ds in list_dataset_replicas(scope='cms', name=dataset, deep=True):
if ds['rse'] == rse:
datasets.update({dataset: ds['available_length']})
return datasets
def update_rule(self):
"""
Adds or removes the rule for the block.
"""
rules = list_replication_rules(filters={'scope': self.scope, 'name': self.block_name})
# rules = self.rcli.list_did_rules(scope=self.scope, name=self.block_name)
rse_expression = 'rse=' + self.rse
remove_rules = [rule for rule in rules
if rule['account'] == self.account and rule['rse_expression'] == rse_expression]
if not remove_rules and self.is_at_pnn:
self.rule_exists = False
if self.dry_run:
logging.info("Dry run: Adding rule for dataset %s at rse %s.", self.block_name, self.rse)
else:
self.add_replication_rule_with_defaults(dids=[{'scope': self.scope, 'name': self.block_name}],
copies=1, rse_expression=rse_expression, account=self.account)
monitor.record_counter('cms_sync.rules_added')
self.rule_exists = True
elif remove_rules and not self.is_at_pnn:
self.rule_exists = True
if self.dry_run:
logging.info("Removing rules for dataset %s at rse %s.", self.block_name, self.rse)
else:
for rule in remove_rules:
# delete_replication_rule(rule['id'], purge_replicas=False, issuer=self.account)
delete_rule(rule_id=rule['id'], purge_replicas=True, soft=False)
monitor.record_counter('cms_sync.rules_removed')
self.rule_exists = False
def GET(self, scope, name):
"""
Return all rules of a given DID.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Not Found
:param scope: The scope name.
"""
header('Content-Type', 'application/x-json-stream')
try:
for rule in list_replication_rules({'scope': scope, 'name': name}):
yield dumps(rule, cls=APIEncoder) + '\n'
except RuleNotFound, error:
raise generate_http_error(404, 'RuleNotFound', error.args[0][0])
def GET(self, account):
"""
Return all rules of a given account.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Not Found
:param scope: The scope name.
"""
header('Content-Type', 'application/x-json-stream')
filters = {'account': account}
if ctx.query:
params = dict(parse_qsl(ctx.query[1:]))
filters.update(params)
try:
for rule in list_replication_rules(filters=filters):
yield dumps(rule, cls=APIEncoder) + '\n'
except RuleNotFound, e:
raise generate_http_error(404, 'RuleNotFound', e.args[0][0])
def generate(vo):
for rule in list_replication_rules(filters=filters, vo=vo):
yield dumps(rule, cls=APIEncoder) + '\n'