def test_list_rules_states(self):
""" SUBSCRIPTION (API): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'},
replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')
subscriptions = list_subscriptions(name=subscription_name, account='root')
# workaround until add_subscription returns the id
id = None
for s in subscriptions:
id = s['id']
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
for r in list_subscription_rule_states(account='root', name=subscription_name):
assert_equal(r[3], 2)
def get(self, account=None, name=None):
"""
Retrieve a subscription.
.. :quickref: Subscription; Get subscriptions.
:param account: The account name.
:param name: The subscription name.
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: Subscription Not Found.
:status 500: Internal Error.
:returns: Line separated list of dictionaries with subscription information.
"""
try:
data = ""
for subscription in list_subscriptions(name=name, account=account):
data += dumps(subscription, cls=APIEncoder) + '\n'
return Response(data, content_type="application/x-json-stream")
except SubscriptionNotFound as error:
return generate_http_error_flask(404, 'SubscriptionNotFound',
error[0][0])
except Exception as error:
return error, 500
def GET(self, account, name):
"""
Return all rules of a given subscription id.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 Internal Error
:param scope: The scope name.
"""
header('Content-Type', 'application/x-json-stream')
state = None
if ctx.query:
params = parse_qs(ctx.query[1:])
if 'state' in params:
state = params['state'][0]
try:
subscriptions = [subscription['id'] for subscription in list_subscriptions(name=name, account=account)]
if len(subscriptions) > 0:
if state:
for rule in list_replication_rules({'subscription_id': subscriptions[0], 'state': state}):
yield dumps(rule, cls=APIEncoder) + '\n'
else:
for rule in list_replication_rules({'subscription_id': subscriptions[0]}):
yield dumps(rule, cls=APIEncoder) + '\n'
except RuleNotFound, error:
raise generate_http_error(404, 'RuleNotFound', error.args[0][0])
def GET(self, account, name):
"""
Return all rules of a given subscription id.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Not Found
:param scope: The scope name.
"""
header('Content-Type', 'application/x-json-stream')
state = None
if ctx.query:
params = parse_qs(ctx.query[1:])
if 'state' in params:
state = params['state'][0]
try:
subscriptions = [subscription['id'] for subscription in list_subscriptions(name=name, account=account)]
if len(subscriptions) > 0:
if state == 'OK':
state = RuleState.OK
if state == 'Replicating':
state = RuleState.REPLICATING
if state == 'Stuck':
state = RuleState.STUCK
for rule in list_replication_rules({'subscription_id': subscriptions[0], 'state': state}):
yield dumps(rule, cls=APIEncoder) + '\n'
except RuleNotFound, e:
raise generate_http_error(404, 'RuleNotFound', e.args[0][0])
def GET(self, account=None, name=None):
"""
Retrieve a subscription.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 Internal Error
406 Not Acceptable
:param account: The account name.
:param name: The subscription name.
"""
header('Content-Type', 'application/x-json-stream')
try:
for subscription in list_subscriptions(name=name,
account=account,
vo=ctx.env.get('vo')):
yield dumps(subscription, cls=APIEncoder) + '\n'
except SubscriptionNotFound as error:
raise generate_http_error(404, 'SubscriptionNotFound',
error.args[0])
except Exception as error:
raise InternalError(error)
def get(self, account, name):
"""
Return all rules of a given subscription id.
.. :quickref: Rules; Get subscription rules.
:param account: The account name.
:param name: The subscription name.
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: Rule Not Found.
:status 404: Subscription Not Found.
:status 406: Not Acceptable.
:returns: Line separated list of dictionaries with rule information.
"""
state = request.args.get('state', default=None)
try:
subscriptions = [subscription['id'] for subscription in list_subscriptions(name=name, account=account, vo=request.environ.get('vo'))]
def generate(vo):
if len(subscriptions) > 0:
if state:
for rule in list_replication_rules({'subscription_id': subscriptions[0], 'state': state}, vo=vo):
yield render_json(**rule) + '\n'
else:
for rule in list_replication_rules({'subscription_id': subscriptions[0]}, vo=vo):
yield render_json(**rule) + '\n'
return try_stream(generate(vo=request.environ.get('vo')))
except (RuleNotFound, SubscriptionNotFound) as error:
return generate_http_error_flask(404, error)
def test_create_and_update_and_list_subscription(self):
""" SUBSCRIPTION (CLIENT): Test the creation of a new subscription, update it, list it """
subscription_name = uuid()
subid = self.client.add_subscription(name=subscription_name, account='root', filter={'project': ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'], 'datatype': ['AOD', ], 'excluded_pattern':
'(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \
\.x|physics_WarmStart|calibration(?!_PixelBeam.merge.(NTUP_IDVTXLUMI|AOD))|merge.HIST|NTUP_MUONCALIB|NTUP_TRIG)', 'account': 'tier0'},
replication_rules=[(2, 'T1_DATATAPE', True, True), (1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='Ni ! Ni!')
result = [sub['id'] for sub in list_subscriptions(name=subscription_name, account='root')]
assert_equal(subid, result[0])
with assert_raises(TypeError):
result = self.client.update_subscription(name=subscription_name, account='root', filter='toto')
result = self.client.update_subscription(name=subscription_name, account='root', filter={'project': ['toto', ]})
assert_true(result)
result = list_subscriptions(name=subscription_name, account='root')
sub = []
for r in result:
sub.append(r)
assert_equal(len(sub), 1)
assert_equal(loads(sub[0]['filter'])['project'][0], 'toto')
def test_create_and_update_and_list_subscription(self):
""" SUBSCRIPTION (CLIENT): Test the creation of a new subscription, update it, list it """
subscription_name = uuid()
with assert_raises(InvalidObject):
subid = self.sub_client.add_subscription(name=subscription_name, account='root', filter={'project': self.projects, 'datatype': ['AOD', ], 'excluded_pattern': self.pattern1, 'account': ['tier0', ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'noactivity'}], lifetime=100000, retroactive=0, dry_run=0, comments='Ni ! Ni!')
subid = self.sub_client.add_subscription(name=subscription_name, account='root', filter={'project': self.projects, 'datatype': ['AOD', ], 'excluded_pattern': self.pattern1, 'account': ['tier0', ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='Ni ! Ni!')
result = [sub['id'] for sub in list_subscriptions(name=subscription_name, account='root')]
assert_equal(subid, result[0])
with assert_raises(TypeError):
result = self.sub_client.update_subscription(name=subscription_name, account='root', filter='toto')
result = self.sub_client.update_subscription(name=subscription_name, account='root', filter={'project': ['toto', ]})
assert_true(result)
result = list_subscriptions(name=subscription_name, account='root')
sub = []
for res in result:
sub.append(res)
assert_equal(len(sub), 1)
assert_equal(loads(sub[0]['filter'])['project'][0], 'toto')
def test_api_subscription(self):
""" SUBSCRIPTION (API): Test external representation of subscriptions """
sub = 'ext_' + generate_uuid()
did = 'ext_' + generate_uuid()
new_acc_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
new_scope_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
add_account(new_acc_name, 'USER', '*****@*****.**', 'root', **self.new_vo)
add_scope(new_scope_name, new_acc_name, 'root', **self.new_vo)
api_acc_lim.set_local_account_limit(new_acc_name, self.rse3_name, 10, 'root', **self.new_vo)
api_acc_lim.set_local_account_limit(new_acc_name, self.rse4_name, 10, 'root', **self.new_vo)
add_did(new_scope_name, did, 'DATASET', 'root', account=new_acc_name, rse=self.rse3_name, **self.new_vo)
sub_id = add_subscription(sub, new_acc_name, {'account': [new_acc_name], 'scope': [new_scope_name]},
[{'copies': 1, 'rse_expression': self.rse3_name, 'weight': 0, 'activity': 'User Subscriptions',
'source_replica_expression': self.rse4_name}],
'', False, 0, 0, 3, 'root', **self.new_vo)
add_replication_rule(dids=[{'scope': new_scope_name, 'name': did}], copies=1, rse_expression=self.rse3_name, weight=None,
lifetime=180, grouping='DATASET', account=new_acc_name, locked=False, subscription_id=sub_id,
source_replica_expression=self.rse4_name, activity='User Subscriptions', notify=None,
purge_replicas=False, ignore_availability=False, comment='', ask_approval=False, asynchronous=False,
priority=0, split_container=False, meta='', issuer='root', **self.new_vo)
out = list_subscriptions(sub, **self.new_vo)
out = list(out)
assert_not_equal(0, len(out))
assert_in(sub_id, [o['id'] for o in out])
for o in out:
if o['id'] == sub_id:
assert_equal(o['account'], new_acc_name)
rules = loads(o['replication_rules'])[0]
assert_equal(rules['rse_expression'], self.rse3_name)
assert_equal(rules['source_replica_expression'], self.rse4_name)
fil = loads(o['filter'])
assert_equal(fil['account'], [new_acc_name])
assert_equal(fil['scope'], [new_scope_name])
out = list_subscription_rule_states(sub, **self.new_vo)
out = list(out)
assert_not_equal(0, len(out))
for o in out:
assert_equal(o.account, new_acc_name)
out = get_subscription_by_id(sub_id, **self.new_vo)
assert_equal(out['account'], new_acc_name)
rules = loads(out['replication_rules'])[0]
assert_equal(rules['rse_expression'], self.rse3_name)
assert_equal(rules['source_replica_expression'], self.rse4_name)
fil = loads(out['filter'])
assert_equal(fil['account'], [new_acc_name])
assert_equal(fil['scope'], [new_scope_name])
def get(self, account, name):
"""
Return all rules of a given subscription id.
.. :quickref: Rules; Get subscription rules.
:param account: The account name.
:param name: The subscription name.
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: Rule Not Found.
:status 404: Subscription Not Found.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: Line separated list of dictionaries with rule information.
"""
state = request.args.get('state', None)
try:
subscriptions = [
subscription['id']
for subscription in list_subscriptions(name=name,
account=account)
]
data = ""
if len(subscriptions) > 0:
if state:
for rule in list_replication_rules({
'subscription_id':
subscriptions[0],
'state':
state
}):
data += dumps(rule, cls=APIEncoder) + '\n'
else:
for rule in list_replication_rules(
{'subscription_id': subscriptions[0]}):
data += dumps(rule, cls=APIEncoder) + '\n'
return Response(data, content_type='application/x-json-stream')
except RuleNotFound as error:
return generate_http_error_flask(404, 'RuleNotFound',
error.args[0])
except SubscriptionNotFound as error:
return generate_http_error_flask(404, 'SubscriptionNotFound',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
return error, 500
def test_list_rules_states(self):
""" SUBSCRIPTION (REST): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
mw = []
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'},
replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery')
subscriptions = list_subscriptions(name=subscription_name, account='root')
# workaround until add_subscription returns the id
id = None
for s in subscriptions:
id = s['id']
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(r1.status, 200)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
r2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True)
for line in r2.body.split('\n'):
print line
rs = loads(line)
if rs[1] == subscription_name:
break
assert_equal(rs[3], 2)
def GET(self, account=None, name=None):
"""
Retrieve a subscription.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Not Found
500 Internal Error
:param account: The account name.
:param name: The subscription name.
"""
header('Content-Type', 'application/x-json-stream')
try:
for subscription in list_subscriptions(name=name, account=account):
yield dumps(subscription, cls=APIEncoder) + '\n'
except SubscriptionNotFound, e:
raise generate_http_error(404, 'SubscriptionNotFound', e[0][0])
def test_create_and_update_and_list_subscription(self):
""" SUBSCRIPTION (API): Test the creation of a new subscription, update it, list it """
subscription_name = uuid()
with assert_raises(InvalidObject):
result = add_subscription(name=subscription_name, account='root', filter={'project': self.projects, 'datatype': ['AOD', ], 'excluded_pattern': self.pattern1, 'account': ['tier0', ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'noactivity'}], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root')
result = add_subscription(name=subscription_name, account='root', filter={'project': self.projects, 'datatype': ['AOD', ], 'excluded_pattern': self.pattern1, 'account': ['tier0', ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root')
with assert_raises(TypeError):
result = update_subscription(name=subscription_name, account='root', metadata={'filter': 'toto'}, issuer='root')
with assert_raises(InvalidObject):
result = update_subscription(name=subscription_name, account='root', metadata={'filter': {'project': 'toto'}}, issuer='root')
result = update_subscription(name=subscription_name, account='root', metadata={'filter': {'project': ['toto', ]}}, issuer='root')
assert_equal(result, None)
result = list_subscriptions(name=subscription_name, account='root')
sub = []
for res in result:
sub.append(res)
assert_equal(len(sub), 1)
assert_equal(loads(sub[0]['filter'])['project'][0], 'toto')
def transmogrifier(worker_number=1, total_workers=1, chunk_size=5, once=False):
"""
Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash, identifies the subscriptions matching the DIDs and submit a replication rule for each DID matching a subscription.
param worker_number: The number of the worker (thread).
param total_number: The total number of workers (threads).
chunk_size: The chunk of the size to process.
once: To run only once
"""
while not graceful_stop.is_set():
dids, subscriptions = [], []
tottime = 0
try:
for did in list_new_dids(worker_number=worker_number, total_workers=total_workers, chunk_size=chunk_size):
d = {"scope": did["scope"], "did_type": str(did["did_type"]), "name": did["name"]}
dids.append(d)
for sub in list_subscriptions(None, None):
if sub["state"] in [SubscriptionState.ACTIVE, SubscriptionState.UPDATED]:
subscriptions.append(sub)
except:
logging.error("Thread %i : Failed to get list of new DIDs or subsscriptions" % (worker_number))
if once:
break
else:
continue
try:
results = {}
start_time = time.time()
logging.debug("Thread %i : In transmogrifier worker" % (worker_number))
identifiers = []
for did in dids:
if did["did_type"] == str(DIDType.DATASET) or did["did_type"] == str(DIDType.CONTAINER):
results["%s:%s" % (did["scope"], did["name"])] = []
try:
metadata = get_metadata(did["scope"], did["name"])
for subscription in subscriptions:
if is_matching_subscription(subscription, did, metadata) is True:
stime = time.time()
results["%s:%s" % (did["scope"], did["name"])].append(subscription["id"])
logging.info(
"Thread %i : %s:%s matches subscription %s"
% (worker_number, did["scope"], did["name"], subscription["name"])
)
for rule in loads(subscription["replication_rules"]):
grouping = rule.get("grouping", "DATASET")
lifetime = rule.get("lifetime", None)
if lifetime:
lifetime = int(lifetime)
weight = rule.get("weight", None)
source_replica_expression = rule.get("source_replica_expression", None)
activity = rule.get("activity", None)
try:
rse_expression = str(rule["rse_expression"]).encode("string-escape")
add_rule(
dids=[{"scope": did["scope"], "name": did["name"]}],
account=subscription["account"],
copies=int(rule["copies"]),
rse_expression=rse_expression,
grouping=grouping,
weight=weight,
lifetime=lifetime,
locked=False,
subscription_id=subscription["id"],
source_replica_expression=source_replica_expression,
activity=activity,
)
monitor.record_counter(counters="transmogrifier.addnewrule.done", delta=1)
if subscription["name"].find("test") > -1:
monitor.record_counter(
counters="transmogrifier.addnewrule.activity.test", delta=1
)
elif subscription["name"].startswith("group"):
monitor.record_counter(
counters="transmogrifier.addnewrule.activity.group", delta=1
)
elif subscription["name"].startswith("tier0export"):
monitor.record_counter(
counters="transmogrifier.addnewrule.activity.tier0export", delta=1
)
elif subscription["name"].endswith("export"):
monitor.record_counter(
counters="transmogrifier.addnewrule.activity.dataconsolidation", delta=1
)
else:
monitor.record_counter(
counters="transmogrifier.addnewrule.activity.other", delta=1
)
except InvalidReplicationRule, e:
logging.error("Thread %i : %s" % (worker_number, str(e)))
monitor.record_counter(
counters="transmogrifier.addnewrule.errortype.InvalidReplicationRule",
delta=1,
)
except InvalidRuleWeight, e:
logging.error("Thread %i : %s" % (worker_number, str(e)))
monitor.record_counter(
counters="transmogrifier.addnewrule.errortype.InvalidRuleWeight", delta=1
)
except InvalidRSEExpression, e:
logging.error("Thread %i : %s" % (worker_number, str(e)))
monitor.record_counter(
counters="transmogrifier.addnewrule.errortype.InvalidRSEExpression", delta=1
)
except StagingAreaRuleRequiresLifetime, e:
logging.error("Thread %i : %s" % (worker_number, str(e)))
monitor.record_counter(
counters="transmogrifier.addnewrule.errortype.StagingAreaRuleRequiresLifetime",
delta=1,
)
except ReplicationRuleCreationTemporaryFailed, e:
# Should never occur. Just for completness
logging.error("Thread %i : %s" % (worker_number, str(e)))
monitor.record_counter(
counters="transmogrifier.addnewrule.errortype.ReplicationRuleCreationTemporaryFailed",
delta=1,
)
def test_create_and_update_and_list_subscription(self):
""" SUBSCRIPTION (API): Test the creation of a new subscription, update it, list it """
rse1, _ = self.rse_factory.make_mock_rse()
rse2, _ = self.rse_factory.make_mock_rse()
rse_expression = '%s|%s' % (rse1, rse2)
subscription_name = uuid()
with pytest.raises(InvalidObject):
result = add_subscription(name=subscription_name,
account='root',
filter_={
'project': self.projects,
'datatype': [
'AOD',
],
'excluded_pattern': self.pattern1,
'account': [
'tier0',
]
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
rse_expression,
'copies':
2,
'activity':
'noactivity'
}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='This is a comment',
issuer='root',
**self.vo)
result = add_subscription(name=subscription_name,
account='root',
filter_={
'project': self.projects,
'datatype': [
'AOD',
],
'excluded_pattern': self.pattern1,
'account': [
'tier0',
]
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
rse_expression,
'copies':
2,
'activity':
'Data Brokering'
}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='This is a comment',
issuer='root',
**self.vo)
with pytest.raises(TypeError):
result = update_subscription(name=subscription_name,
account='root',
metadata={'filter': 'toto'},
issuer='root',
**self.vo)
with pytest.raises(InvalidObject):
result = update_subscription(
name=subscription_name,
account='root',
metadata={'filter': {
'project': 'toto'
}},
issuer='root',
**self.vo)
result = update_subscription(
name=subscription_name,
account='root',
metadata={'filter': {
'project': [
'toto',
]
}},
issuer='root',
**self.vo)
assert result is None
result = list_subscriptions(name=subscription_name,
account='root',
**self.vo)
sub = []
for res in result:
sub.append(res)
assert len(sub) == 1
assert loads(sub[0]['filter'])['project'][0] == 'toto'
def test_create_and_update_and_list_subscription(self):
""" SUBSCRIPTION (CLIENT): Test the creation of a new subscription, update it, list it """
subscription_name = uuid()
rse1, _ = self.rse_factory.make_mock_rse()
rse2, _ = self.rse_factory.make_mock_rse()
rse_expression = '%s|%s' % (rse1, rse2)
with pytest.raises(InvalidObject):
subid = self.sub_client.add_subscription(name=subscription_name,
account='root',
filter_={
'project':
self.projects,
'datatype': [
'AOD',
],
'excluded_pattern':
self.pattern1,
'account': [
'tier0',
]
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
rse_expression,
'copies':
2,
'activity':
'noactivity'
}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='Ni ! Ni!')
subid = self.sub_client.add_subscription(name=subscription_name,
account='root',
filter_={
'project': self.projects,
'datatype': [
'AOD',
],
'excluded_pattern':
self.pattern1,
'account': [
'tier0',
]
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
rse_expression,
'copies':
2,
'activity':
'Data Brokering'
}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='Ni ! Ni!')
result = [
sub['id'] for sub in list_subscriptions(
name=subscription_name, account='root', **self.vo)
]
assert subid == result[0]
with pytest.raises(TypeError):
result = self.sub_client.update_subscription(
name=subscription_name, account='root', filter_='toto')
result = self.sub_client.update_subscription(
name=subscription_name,
account='root',
filter_={'project': [
'toto',
]})
assert result
result = list_subscriptions(name=subscription_name,
account='root',
**self.vo)
sub = []
for res in result:
sub.append(res)
assert len(sub) == 1
assert loads(sub[0]['filter'])['project'][0] == 'toto'
def transmogrifier(bulk=5, once=False):
"""
Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash,
identifies the subscriptions matching the DIDs and
submit a replication rule for each DID matching a subscription.
:param thread: Thread number at startup.
:param bulk: The number of requests to process.
:param once: Run only once.
"""
executable = ' '.join(argv)
hostname = socket.getfqdn()
pid = os.getpid()
hb_thread = threading.current_thread()
heartbeat.sanity_check(executable=executable, hostname=hostname)
while not graceful_stop.is_set():
heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
dids, subscriptions = [], []
tottime = 0
prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'] + 1,
heart_beat['nr_threads'])
try:
for did in list_new_dids(thread=heart_beat['assign_thread'],
total_threads=heart_beat['nr_threads'],
chunk_size=bulk):
dids.append({
'scope': did['scope'],
'did_type': str(did['did_type']),
'name': did['name']
})
sub_dict = {3: []}
for sub in list_subscriptions(None, None):
if sub['state'] != SubscriptionState.INACTIVE and sub[
'lifetime'] and (datetime.now() > sub['lifetime']):
update_subscription(name=sub['name'],
account=sub['account'],
state=SubscriptionState.INACTIVE)
elif sub['state'] in [
SubscriptionState.ACTIVE, SubscriptionState.UPDATED
]:
priority = 3
if 'policyid' in sub:
if int(sub['policyid']) not in sub_dict:
sub_dict[int(sub['policyid'])] = []
priority = int(sub['policyid'])
sub_dict[priority].append(sub)
priorities = sub_dict.keys()
priorities.sort()
for priority in priorities:
subscriptions.extend(sub_dict[priority])
except SubscriptionNotFound, error:
logging.warning(prepend_str + 'No subscriptions defined: %s' %
(str(error)))
time.sleep(10)
continue
except Exception, error:
logging.error(
prepend_str +
'Failed to get list of new DIDs or subscriptions: %s' %
(str(error)))
if once:
break
else:
continue
def transmogrifier(bulk=5, once=False, sleep_time=60):
"""
Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash,
identifies the subscriptions matching the DIDs and
submit a replication rule for each DID matching a subscription.
:param thread: Thread number at startup.
:param bulk: The number of requests to process.
:param once: Run only once.
:param sleep_time: Time between two cycles.
"""
executable = ' '.join(argv)
hostname = socket.getfqdn()
pid = os.getpid()
hb_thread = threading.current_thread()
heartbeat.sanity_check(executable=executable, hostname=hostname)
while not graceful_stop.is_set():
heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
dids, subscriptions = [], []
tottime = 0
prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'] + 1,
heart_beat['nr_threads'])
try:
# Get the new DIDs based on the is_new flag
for did in list_new_dids(thread=heart_beat['assign_thread'],
total_threads=heart_beat['nr_threads'],
chunk_size=bulk):
dids.append({
'scope': did['scope'],
'did_type': str(did['did_type']),
'name': did['name']
})
sub_dict = {3: []}
# Get the list of subscriptions. The default priority of the subscription is 3. 0 is the highest priority, 5 the lowest
# The priority is defined as 'policyid'
for sub in list_subscriptions(None, None):
if sub['state'] != SubscriptionState.INACTIVE and sub[
'lifetime'] and (datetime.now() > sub['lifetime']):
update_subscription(
name=sub['name'],
account=sub['account'],
metadata={'state': SubscriptionState.INACTIVE},
issuer='root')
elif sub['state'] in [
SubscriptionState.ACTIVE, SubscriptionState.UPDATED
]:
priority = 3
if 'policyid' in sub:
if int(sub['policyid']) not in sub_dict:
sub_dict[int(sub['policyid'])] = []
priority = int(sub['policyid'])
sub_dict[priority].append(sub)
priorities = list(sub_dict.keys())
priorities.sort()
# Order the subscriptions according to their priority
for priority in priorities:
subscriptions.extend(sub_dict[priority])
except SubscriptionNotFound as error:
logging.warning(prepend_str + 'No subscriptions defined: %s' %
(str(error)))
time.sleep(10)
continue
except Exception as error:
logging.error(
prepend_str +
'Failed to get list of new DIDs or subscriptions: %s' %
(str(error)))
try:
results = {}
start_time = time.time()
blacklisted_rse_id = [
rse['id'] for rse in list_rses({'availability_write': False})
]
logging.debug(prepend_str + 'In transmogrifier worker')
identifiers = []
# Loop over all the new dids
for did in dids:
did_success = True
if did['did_type'] == str(
DIDType.DATASET) or did['did_type'] == str(
DIDType.CONTAINER):
results['%s:%s' % (did['scope'], did['name'])] = []
try:
metadata = get_metadata(did['scope'], did['name'])
# Loop over all the subscriptions
for subscription in subscriptions:
# Check if the DID match the subscription
if is_matching_subscription(
subscription, did, metadata) is True:
filter_string = loads(subscription['filter'])
split_rule = filter_string.get(
'split_rule', False)
if split_rule == 'true':
split_rule = True
elif split_rule == 'false':
split_rule = False
stime = time.time()
results['%s:%s' %
(did['scope'], did['name'])].append(
subscription['id'])
logging.info(prepend_str +
'%s:%s matches subscription %s' %
(did['scope'], did['name'],
subscription['name']))
for rule_string in loads(
subscription['replication_rules']):
# Get all the rule and subscription parameters
grouping = rule_string.get(
'grouping', 'DATASET')
lifetime = rule_string.get(
'lifetime', None)
ignore_availability = rule_string.get(
'ignore_availability', None)
weight = rule_string.get('weight', None)
source_replica_expression = rule_string.get(
'source_replica_expression', None)
locked = rule_string.get('locked', None)
if locked == 'True':
locked = True
else:
locked = False
purge_replicas = rule_string.get(
'purge_replicas', False)
if purge_replicas == 'True':
purge_replicas = True
else:
purge_replicas = False
rse_expression = str(
rule_string['rse_expression'])
comment = str(subscription['comments'])
subscription_id = str(subscription['id'])
account = subscription['account']
copies = int(rule_string['copies'])
activity = rule_string.get(
'activity', 'User Subscriptions')
try:
validate_schema(name='activity',
obj=activity)
except InputValidationError as error:
logging.error(
prepend_str +
'Error validating the activity %s'
% (str(error)))
activity = 'User Subscriptions'
if lifetime:
lifetime = int(lifetime)
str_activity = "".join(activity.split())
success = False
nattempt = 5
attemptnr = 0
skip_rule_creation = False
if split_rule:
rses = parse_expression(rse_expression)
list_of_rses = [
rse['rse'] for rse in rses
]
# Check that some rule doesn't already exist for this DID and subscription
preferred_rse_ids = []
for rule in list_rules(
filters={
'subscription_id':
subscription_id,
'scope': did['scope'],
'name': did['name']
}):
already_existing_rses = [
(rse['rse'], rse['id'])
for rse in parse_expression(
rule['rse_expression'])
]
for rse, rse_id in already_existing_rses:
if (rse in list_of_rses) and (
rse_id not in
preferred_rse_ids):
preferred_rse_ids.append(
rse_id)
if len(preferred_rse_ids) >= copies:
skip_rule_creation = True
rse_id_dict = {}
for rse in rses:
rse_id_dict[rse['id']] = rse['rse']
try:
rseselector = RSESelector(
account=account,
rses=rses,
weight=weight,
copies=copies -
len(preferred_rse_ids))
selected_rses = [
rse_id_dict[rse_id] for rse_id,
_, _ in rseselector.select_rse(
0,
preferred_rse_ids=
preferred_rse_ids,
copies=copies,
blacklist=blacklisted_rse_id
)
]
except (InsufficientTargetRSEs,
InsufficientAccountLimit,
InvalidRuleWeight,
RSEOverQuota) as error:
logging.warning(
prepend_str +
'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blacklisted sites'
% (subscription['name'],
account, str(error)))
# Now including the blacklisted sites
try:
rseselector = RSESelector(
account=account,
rses=rses,
weight=weight,
copies=copies -
len(preferred_rse_ids))
selected_rses = [
rse_id_dict[rse_id]
for rse_id, _, _ in
rseselector.select_rse(
0,
preferred_rse_ids=
preferred_rse_ids,
copies=copies,
blacklist=[])
]
ignore_availability = True
except (InsufficientTargetRSEs,
InsufficientAccountLimit,
InvalidRuleWeight,
RSEOverQuota) as error:
logging.error(
prepend_str +
'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.'
% (subscription['name'],
account, str(error)))
monitor.record_counter(
counters=
'transmogrifier.addnewrule.errortype.%s'
% (str(error.__class__.
__name__)),
delta=1)
# The DID won't be reevaluated at the next cycle
did_success = did_success and True
continue
for attempt in range(0, nattempt):
attemptnr = attempt
nb_rule = 0
# Try to create the rule
try:
if split_rule:
if not skip_rule_creation:
for rse in selected_rses:
logging.info(
prepend_str +
'Will insert one rule for %s:%s on %s'
%
(did['scope'],
did['name'], rse))
add_rule(
dids=[{
'scope':
did['scope'],
'name':
did['name']
}],
account=account,
copies=1,
rse_expression=rse,
grouping=grouping,
weight=weight,
lifetime=lifetime,
locked=locked,
subscription_id=
subscription_id,
source_replica_expression
=source_replica_expression,
activity=activity,
purge_replicas=
purge_replicas,
ignore_availability=
ignore_availability,
comment=comment)
nb_rule += 1
if nb_rule == copies:
success = True
break
else:
add_rule(
dids=[{
'scope': did['scope'],
'name': did['name']
}],
account=account,
copies=copies,
rse_expression=
rse_expression,
grouping=grouping,
weight=weight,
lifetime=lifetime,
locked=locked,
subscription_id=
subscription['id'],
source_replica_expression=
source_replica_expression,
activity=activity,
purge_replicas=
purge_replicas,
ignore_availability=
ignore_availability,
comment=comment)
nb_rule += 1
monitor.record_counter(
counters=
'transmogrifier.addnewrule.done',
delta=nb_rule)
monitor.record_counter(
counters=
'transmogrifier.addnewrule.activity.%s'
% str_activity,
delta=nb_rule)
success = True
break
except (InvalidReplicationRule,
InvalidRuleWeight,
InvalidRSEExpression,
StagingAreaRuleRequiresLifetime,
DuplicateRule) as error:
# Errors that won't be retried
success = True
logging.error(prepend_str + '%s' %
(str(error)))
monitor.record_counter(
counters=
'transmogrifier.addnewrule.errortype.%s'
% (str(
error.__class__.__name__)),
delta=1)
break
except (ReplicationRuleCreationTemporaryFailed,
InsufficientTargetRSEs,
InsufficientAccountLimit,
DatabaseException,
RSEBlacklisted) as error:
# Errors to be retried
logging.error(
prepend_str +
'%s Will perform an other attempt %i/%i'
% (str(error), attempt + 1,
nattempt))
monitor.record_counter(
counters=
'transmogrifier.addnewrule.errortype.%s'
% (str(
error.__class__.__name__)),
delta=1)
except Exception as error:
# Unexpected errors
monitor.record_counter(
counters=
'transmogrifier.addnewrule.errortype.unknown',
delta=1)
exc_type, exc_value, exc_traceback = exc_info(
)
logging.critical(
prepend_str + ''.join(
format_exception(
exc_type, exc_value,
exc_traceback)).strip(
))
did_success = (did_success and success)
if (attemptnr +
1) == nattempt and not success:
logging.error(
prepend_str +
'Rule for %s:%s on %s cannot be inserted'
% (did['scope'], did['name'],
rse_expression))
else:
logging.info(
prepend_str +
'%s rule(s) inserted in %f seconds'
% (str(nb_rule),
time.time() - stime))
except DataIdentifierNotFound as error:
logging.warning(prepend_str + error)
if did_success:
if did['did_type'] == str(DIDType.FILE):
monitor.record_counter(
counters='transmogrifier.did.file.processed',
delta=1)
elif did['did_type'] == str(DIDType.DATASET):
monitor.record_counter(
counters='transmogrifier.did.dataset.processed',
delta=1)
elif did['did_type'] == str(DIDType.CONTAINER):
monitor.record_counter(
counters='transmogrifier.did.container.processed',
delta=1)
monitor.record_counter(
counters='transmogrifier.did.processed', delta=1)
identifiers.append({
'scope':
did['scope'],
'name':
did['name'],
'did_type':
DIDType.from_sym(did['did_type'])
})
time1 = time.time()
# Mark the DIDs as processed
for identifier in chunks(identifiers, 100):
_retrial(set_new_dids, identifier, None)
logging.info(prepend_str + 'Time to set the new flag : %f' %
(time.time() - time1))
tottime = time.time() - start_time
for sub in subscriptions:
update_subscription(
name=sub['name'],
account=sub['account'],
metadata={'last_processed': datetime.now()},
issuer='root')
logging.info(prepend_str +
'It took %f seconds to process %i DIDs' %
(tottime, len(dids)))
logging.debug(prepend_str + 'DIDs processed : %s' % (str(dids)))
monitor.record_counter(counters='transmogrifier.job.done', delta=1)
monitor.record_timer(stat='transmogrifier.job.duration',
time=1000 * tottime)
except Exception:
exc_type, exc_value, exc_traceback = exc_info()
logging.critical(prepend_str + ''.join(
format_exception(exc_type, exc_value, exc_traceback)).strip())
monitor.record_counter(counters='transmogrifier.job.error',
delta=1)
monitor.record_counter(counters='transmogrifier.addnewrule.error',
delta=1)
if once is True:
break
if tottime < sleep_time:
logging.info(prepend_str + 'Will sleep for %s seconds' %
(sleep_time - tottime))
time.sleep(sleep_time - tottime)
heartbeat.die(executable, hostname, pid, hb_thread)
logging.info(prepend_str + 'Graceful stop requested')
logging.info(prepend_str + 'Graceful stop done')
def test_subscriptions_at_different_vos(self):
""" MULTI VO (CLIENT): Test that subscriptions from 2nd vo don't interfere """
account_client = AccountClient()
usr_uuid = str(generate_uuid()).lower()[:16]
shr_acc = 'shr-%s' % usr_uuid
account_client.add_account(shr_acc, 'USER', '*****@*****.**')
add_account(shr_acc, 'USER', '*****@*****.**', 'root', **self.new_vo)
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
tst_scope = 'tst_%s' % scope_uuid
new_scope = 'new_%s' % scope_uuid
scope_client.add_scope('root', tst_scope)
add_scope(new_scope, 'root', 'root', **self.new_vo)
did_client = DIDClient()
did_uuid = str(generate_uuid()).lower()[:16]
tst_did = 'tstset_%s' % did_uuid
new_did = 'newset_%s' % did_uuid
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst_rse1 = 'TST1_%s' % rse_str
tst_rse2 = 'TST2_%s' % rse_str
new_rse1 = 'NEW1_%s' % rse_str
new_rse2 = 'NEW2_%s' % rse_str
rse_client.add_rse(tst_rse1)
rse_client.add_rse(tst_rse2)
add_rse(new_rse1, 'root', **self.new_vo)
add_rse(new_rse2, 'root', **self.new_vo)
acc_lim_client = AccountLimitClient()
acc_lim_client.set_local_account_limit(shr_acc, tst_rse1, 10)
acc_lim_client.set_local_account_limit(shr_acc, tst_rse2, 10)
set_local_account_limit(shr_acc, new_rse1, 10, 'root', **self.new_vo)
set_local_account_limit(shr_acc, new_rse2, 10, 'root', **self.new_vo)
did_client.add_did(tst_scope, tst_did, 'DATASET', rse=tst_rse1)
add_did(new_scope, new_did, 'DATASET', 'root', rse=new_rse1, **self.new_vo)
sub_client = SubscriptionClient()
sub_str = generate_uuid()
tst_sub = 'tstsub_' + sub_str
new_sub = 'newsub_' + sub_str
shr_sub = 'shrsub_' + sub_str
tst_sub_id = sub_client.add_subscription(tst_sub, shr_acc, {'scope': [tst_scope]},
[{'copies': 1, 'rse_expression': tst_rse2, 'weight': 0,
'activity': 'User Subscriptions'}],
'', None, 0, 0)
shr_tst_sub_id = sub_client.add_subscription(shr_sub, shr_acc, {'scope': [tst_scope]},
[{'copies': 1, 'rse_expression': tst_rse2, 'weight': 0,
'activity': 'User Subscriptions'}],
'', None, 0, 0)
new_sub_id = add_subscription(new_sub, shr_acc, {'scope': [new_scope]},
[{'copies': 1, 'rse_expression': new_rse2, 'weight': 0, 'activity': 'User Subscriptions'}],
'', False, 0, 0, 3, 'root', **self.new_vo)
shr_new_sub_id = add_subscription(shr_sub, shr_acc, {'scope': [new_scope]},
[{'copies': 1, 'rse_expression': new_rse2, 'weight': 0, 'activity': 'User Subscriptions'}],
'', False, 0, 0, 3, 'root', **self.new_vo)
tst_subs = [s['id'] for s in sub_client.list_subscriptions()]
assert_in(tst_sub_id, tst_subs)
assert_in(shr_tst_sub_id, tst_subs)
assert_not_in(new_sub_id, tst_subs)
assert_not_in(shr_new_sub_id, tst_subs)
new_subs = [s['id'] for s in list_subscriptions(**self.new_vo)]
assert_in(new_sub_id, new_subs)
assert_in(shr_new_sub_id, new_subs)
assert_not_in(tst_sub_id, new_subs)
assert_not_in(shr_tst_sub_id, new_subs)
shr_tst_subs = [s['id'] for s in sub_client.list_subscriptions(name=shr_sub)]
assert_in(shr_tst_sub_id, shr_tst_subs)
assert_not_in(shr_new_sub_id, shr_tst_subs)
shr_new_subs = [s['id'] for s in list_subscriptions(name=shr_sub, **self.new_vo)]
assert_in(shr_new_sub_id, shr_new_subs)
assert_not_in(shr_tst_sub_id, shr_new_subs)
acc_tst_subs = [s['id'] for s in sub_client.list_subscriptions(account=shr_acc)]
assert_in(tst_sub_id, acc_tst_subs)
assert_in(shr_tst_sub_id, acc_tst_subs)
assert_not_in(new_sub_id, acc_tst_subs)
assert_not_in(shr_new_sub_id, acc_tst_subs)
acc_new_subs = [s['id'] for s in list_subscriptions(account=shr_acc, **self.new_vo)]
assert_in(new_sub_id, acc_new_subs)
assert_in(shr_new_sub_id, acc_new_subs)
assert_not_in(tst_sub_id, acc_new_subs)
assert_not_in(shr_tst_sub_id, acc_new_subs)
def generate(vo):
for subscription in list_subscriptions(name=name,
account=account,
vo=vo):
yield render_json(**subscription) + '\n'
