def add_replication_rule_with_defaults(self,dids, copies, rse_expression, account):
"""
Add replication rule requires one to send all the values. Add a list of defaults.
If true options are required, move them into the parameter list.
:param dids: List of dids (scope/name dictionary)
:param copies: Number of copies
:param rse_expression: RSE expression
:param account: Account for the rule
:return: None
"""
(grouping, weight, lifetime, locked, subscription_id, source_replica_expression, notify,
purge_replicas, ignore_availability, comment, ask_approval, asynchronous, priority, split_container) = (
'DATASET', None, None, False, None, None, None, False, False, None, False, False, 3, False)
activity = 'Data Consolidation'
meta = json.dumps({"phedex_group": self.group, "phedex_custodial": self.custodial})
add_replication_rule(dids=dids, copies=copies, rse_expression=rse_expression, account=account,
grouping=grouping, weight=weight, lifetime=lifetime, locked=locked,
subscription_id=subscription_id, source_replica_expression=source_replica_expression,
activity=activity, notify=notify, purge_replicas=purge_replicas,
ignore_availability=ignore_availability, comment=comment, ask_approval=ask_approval,
asynchronous=asynchronous, priority=priority, split_container=split_container, meta=meta,
issuer=account)
def populateDB():
listrses = list_rses({'T1': '1'})
print len(listrses), listrses
# listrses = list_rses()
# print len(listrses), listrses
# sys.exit()
account = 'root'
project = 'mc12_8TeV'
dictDistrib = [{'datatype': 'HITS', 'prodstep': 'merge', 'nbfiles': 302, 'totfilesize': 225394185112, 'nbreplicas': 1}, {'datatype': 'HITS', 'prodstep': 'simul', 'nbfiles': 620, 'totfilesize': 97930909866, 'nbreplicas': 1},
{'datatype': 'EVNT', 'prodstep': 'evgen', 'nbfiles': 324, 'totfilesize': 7809298802, 'nbreplicas': 3}, {'datatype': 'AOD', 'prodstep': 'merge', 'nbfiles': 52, 'totfilesize': 106942334943, 'nbreplicas': 4},
{'datatype': 'AOD', 'prodstep': 'recon', 'nbfiles': 858, 'totfilesize': 182186965627, 'nbreplicas': 1}]
for d in dictDistrib:
for day in xrange(0, 180):
for i in xrange(0, 30):
scope = project
prod_step = d['prodstep']
datatype = d['datatype']
nbfiles = int(d['nbfiles'])
filesize = int(int(d['totfilesize'])/float(nbfiles))
nbfiles = int(random.gauss(nbfiles, nbfiles/10))
filesize = int(random.gauss(filesize, filesize/10))
nbreplicas = int(d['nbreplicas'])
dataset_meta = {'project': project, 'stream_name': 'dummy', 'prod_step': prod_step, 'datatype': datatype}
source_rses = []
if nbreplicas:
iter = 0
while (len(source_rses) != nbreplicas and iter != 100):
rnd_site = random.choice(listrses)
iter += 1
if rnd_site not in source_rses:
source_rses.append(rnd_site)
try:
dsn = '%s.%s.%s.%i.%i' % (project, prod_step, datatype, day, i)
print '%i Creating %s with %i files of size %i located at %i sites' % (i, dsn, nbfiles, filesize, len(source_rses))
add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True}, meta=dataset_meta)
files = ['file_%s' % uuid() for i in xrange(nbfiles)]
listfiles = []
for file in files:
listfiles.append({'scope': scope, 'name': file, 'size': filesize})
for source_rse in source_rses:
add_file_replica(source_rse, scope, file, filesize, issuer=account)
attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
for source_rse in source_rses:
try:
add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=source_rse,
grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer='root')
except InvalidReplicationRule, e:
print e
except RucioException, e:
print e
def populateDB():
listrses = list_rses({'T1': '1'})
print(len(listrses), listrses)
# listrses = list_rses()
# print len(listrses), listrses
# sys.exit()
account = 'root'
project = 'mc12_8TeV'
dictDistrib = [{'datatype': 'HITS', 'prodstep': 'merge', 'nbfiles': 302, 'totfilesize': 225394185112, 'nbreplicas': 1}, {'datatype': 'HITS', 'prodstep': 'simul', 'nbfiles': 620, 'totfilesize': 97930909866, 'nbreplicas': 1},
{'datatype': 'EVNT', 'prodstep': 'evgen', 'nbfiles': 324, 'totfilesize': 7809298802, 'nbreplicas': 3}, {'datatype': 'AOD', 'prodstep': 'merge', 'nbfiles': 52, 'totfilesize': 106942334943, 'nbreplicas': 4},
{'datatype': 'AOD', 'prodstep': 'recon', 'nbfiles': 858, 'totfilesize': 182186965627, 'nbreplicas': 1}]
for d in dictDistrib:
for day in range(180):
for i in range(30):
scope = project
prod_step = d['prodstep']
datatype = d['datatype']
nbfiles = int(d['nbfiles'])
filesize = int(int(d['totfilesize']) / float(nbfiles))
nbfiles = int(random.gauss(nbfiles, nbfiles / 10))
filesize = int(random.gauss(filesize, filesize / 10))
nbreplicas = int(d['nbreplicas'])
dataset_meta = {'project': project, 'stream_name': 'dummy', 'prod_step': prod_step, 'datatype': datatype}
source_rses = []
if nbreplicas:
iter = 0
while (len(source_rses) != nbreplicas and iter != 100):
rnd_site = random.choice(listrses)
iter += 1
if rnd_site not in source_rses:
source_rses.append(rnd_site)
try:
dsn = '%s.%s.%s.%i.%i' % (project, prod_step, datatype, day, i)
print('%i Creating %s with %i files of size %i located at %i sites' % (i, dsn, nbfiles, filesize, len(source_rses)))
add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True}, meta=dataset_meta)
files = ['file_%s' % uuid() for i in range(nbfiles)]
listfiles = []
for file in files:
listfiles.append({'scope': scope, 'name': file, 'size': filesize})
for source_rse in source_rses:
add_file_replica(source_rse, scope, file, filesize, issuer=account)
attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
for source_rse in source_rses:
try:
add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=source_rse,
grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer='root')
except InvalidReplicationRule as e:
print(e)
except RucioException as e:
print(e)
def populateDB(filename=None):
listrses = list_rses(filters={'deterministic': 1})
listrses = map(lambda x: x['rse'], listrses)
account = 'root'
pdf = generatePDF()
# Generate 200000 datasets according to the dataset distribution
for index in range(20000):
scope_nb = getRandomScope(pdf)
project = 'user.user%i' % (scope_nb)
scope = 'user.user%i' % (scope_nb)
account = 'user%i' % (scope_nb)
print(scope)
nbfiles = 53
filesize = 78000000
uid = uuid()
dsn = '%s.%s' % (project, uid)
rnd_site = random.choice(listrses)
print('%i Creating %s with %i files of size %i located at %s' %
(index, dsn, nbfiles, filesize, rnd_site))
add_identifier(scope=scope,
name=dsn,
type='dataset',
issuer=account,
statuses={'monotonic': True})
monitor.record(timeseries='dbfiller.addnewdataset', delta=1)
files = ['file_%s' % uuid() for i in range(nbfiles)]
listfiles = []
for file in files:
listfiles.append({'scope': scope, 'name': file, 'size': filesize})
add_file_replica(rnd_site, scope, file, filesize, issuer=account)
monitor.record(timeseries='dbfiller.addreplicas', delta=nbfiles)
attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
monitor.record(timeseries='dbfiller.addnewfile', delta=nbfiles)
try:
add_replication_rule(dids=[{
'scope': scope,
'name': dsn
}],
account=account,
copies=1,
rse_expression=rnd_site,
grouping='DATASET',
weight=None,
lifetime=None,
locked=False,
subscription_id=None,
issuer=account)
monitor.record(timeseries='dbfiller.addreplicationrules', delta=1)
except InvalidReplicationRule as e:
print(e)
def test_api_subscription(self):
""" SUBSCRIPTION (API): Test external representation of subscriptions """
sub = 'ext_' + generate_uuid()
did = 'ext_' + generate_uuid()
new_acc_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
new_scope_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
add_account(new_acc_name, 'USER', '*****@*****.**', 'root', **self.new_vo)
add_scope(new_scope_name, new_acc_name, 'root', **self.new_vo)
api_acc_lim.set_local_account_limit(new_acc_name, self.rse3_name, 10, 'root', **self.new_vo)
api_acc_lim.set_local_account_limit(new_acc_name, self.rse4_name, 10, 'root', **self.new_vo)
add_did(new_scope_name, did, 'DATASET', 'root', account=new_acc_name, rse=self.rse3_name, **self.new_vo)
sub_id = add_subscription(sub, new_acc_name, {'account': [new_acc_name], 'scope': [new_scope_name]},
[{'copies': 1, 'rse_expression': self.rse3_name, 'weight': 0, 'activity': 'User Subscriptions',
'source_replica_expression': self.rse4_name}],
'', False, 0, 0, 3, 'root', **self.new_vo)
add_replication_rule(dids=[{'scope': new_scope_name, 'name': did}], copies=1, rse_expression=self.rse3_name, weight=None,
lifetime=180, grouping='DATASET', account=new_acc_name, locked=False, subscription_id=sub_id,
source_replica_expression=self.rse4_name, activity='User Subscriptions', notify=None,
purge_replicas=False, ignore_availability=False, comment='', ask_approval=False, asynchronous=False,
priority=0, split_container=False, meta='', issuer='root', **self.new_vo)
out = list_subscriptions(sub, **self.new_vo)
out = list(out)
assert_not_equal(0, len(out))
assert_in(sub_id, [o['id'] for o in out])
for o in out:
if o['id'] == sub_id:
assert_equal(o['account'], new_acc_name)
rules = loads(o['replication_rules'])[0]
assert_equal(rules['rse_expression'], self.rse3_name)
assert_equal(rules['source_replica_expression'], self.rse4_name)
fil = loads(o['filter'])
assert_equal(fil['account'], [new_acc_name])
assert_equal(fil['scope'], [new_scope_name])
out = list_subscription_rule_states(sub, **self.new_vo)
out = list(out)
assert_not_equal(0, len(out))
for o in out:
assert_equal(o.account, new_acc_name)
out = get_subscription_by_id(sub_id, **self.new_vo)
assert_equal(out['account'], new_acc_name)
rules = loads(out['replication_rules'])[0]
assert_equal(rules['rse_expression'], self.rse3_name)
assert_equal(rules['source_replica_expression'], self.rse4_name)
fil = loads(out['filter'])
assert_equal(fil['account'], [new_acc_name])
assert_equal(fil['scope'], [new_scope_name])
def populateDB(filename=None):
listrses = list_rses(filters={'deterministic': 1})
listrses = map(lambda x: x['rse'], listrses)
account = 'root'
pdf = generatePDF()
# Generate 200000 datasets according to the dataset distribution
for index in xrange(0, 20000):
scope_nb = getRandomScope(pdf)
project = 'user.user%i' % (scope_nb)
scope = 'user.user%i' % (scope_nb)
account = 'user%i' % (scope_nb)
print scope
nbfiles = 53
filesize = 78000000
uid = uuid()
dsn = '%s.%s' % (project, uid)
rnd_site = random.choice(listrses)
print '%i Creating %s with %i files of size %i located at %s' % (index, dsn, nbfiles, filesize, rnd_site)
add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True})
monitor.record(timeseries='dbfiller.addnewdataset', delta=1)
files = ['file_%s' % uuid() for i in xrange(nbfiles)]
listfiles = []
for file in files:
listfiles.append({'scope': scope, 'name': file, 'size': filesize})
add_file_replica(rnd_site, scope, file, filesize, issuer=account)
monitor.record(timeseries='dbfiller.addreplicas', delta=nbfiles)
attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
monitor.record(timeseries='dbfiller.addnewfile', delta=nbfiles)
try:
add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=rnd_site,
grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer=account)
monitor.record(timeseries='dbfiller.addreplicationrules', delta=1)
except InvalidReplicationRule, e:
print e
def populateDB(filename=None):
listrses = list_rses(filters={'deterministic': 1})
print listrses
listrses = map(lambda x: x['rse'], listrses)
account = 'root'
nbDatasets = 0
list = []
dictDistrib = {}
if not filename:
if os.getenv('RUCIO_HOME'):
filename = os.getenv(
'RUCIO_HOME') + '/etc/data12_8TeV_distribution.txt'
else:
filename = '/opt/rucio/etc/data12_8TeV_distribution.txt'
# Get the dataset distribution
f = open(filename, 'r')
for line in f:
if not line.startswith('NBDATASETS'):
line = line.rstrip('\n')
strsplit = line.split()
dictDistrib[(nbDatasets,
nbDatasets + int(strsplit[0]))] = strsplit[1:]
nbDatasets += int(strsplit[0])
list.append([
nbDatasets,
] + strsplit[1:])
f.close()
# Generate 200000 datasets according to the dataset distribution
for i in xrange(0, 200000):
rnd = random.random() * nbDatasets
for lower, upper in dictDistrib:
if (rnd > lower) and (rnd < upper):
project = dictDistrib[lower, upper][0]
scope = project
run_number = random.randint(0, 1000000)
tag = random.randint(0, 10000)
stream_name = dictDistrib[lower, upper][1]
prod_step = dictDistrib[lower, upper][2]
datatype = dictDistrib[lower, upper][3]
provenance = dictDistrib[lower, upper][4]
group = dictDistrib[lower, upper][5]
if group == '/atlas/role=production':
# account = 'atlasprod'
account = 'panda'
if provenance == 'T0':
group = 'tier0'
account = 'tier0'
else:
group = 'panda'
else:
# account = dictGroups[group]
account = 'panda'
scope = 'group.%s' % (dictGroups[group])
group = dictGroups[group]
nbfiles = int(dictDistrib[lower, upper][6])
filesize = int(
int(dictDistrib[lower, upper][7]) / float(nbfiles))
nbreplicas = int(dictDistrib[lower, upper][8])
if group == 'panda' or group == 'tier0':
dataset_meta = {
'project': project,
'run_number': run_number,
'stream_name': stream_name,
'prod_step': prod_step,
'datatype': datatype,
'provenance': provenance,
'group': group
}
else:
campaign = int(tag / 1000.)
dataset_meta = {
'project': project,
'run_number': run_number,
'stream_name': stream_name,
'prod_step': prod_step,
'datatype': datatype,
'provenance': provenance,
'group': group,
'campaign': '%s_repro_%i' % (group, campaign)
}
source_rses = []
if nbreplicas:
iter = 0
while (len(source_rses) != nbreplicas and iter != 100):
rnd_site = random.choice(listrses)
iter += 1
if (rnd_site not in source_rses):
source_rses.append(rnd_site)
run_number_string = str(run_number)
run_number_string = run_number_string.rjust(7, '0')
dsn = '%s.%s.%s.%s.%s.%s' % (project, run_number_string,
stream_name, prod_step,
datatype, tag)
print '%i Creating %s:%s with %i files of size %i located at %i sites' % (
i, scope, dsn, nbfiles, filesize, len(source_rses))
stime1 = time.time()
add_identifier(scope=scope,
name=dsn,
type='dataset',
issuer=account,
statuses={'monotonic': True},
meta=dataset_meta)
stime2 = time.time()
print 'Time to generate a dataset : %s' % str(stime2 -
stime1)
monitor.record(timeseries='dbfiller.addnewdataset',
delta=1)
files = ['file_%s' % uuid() for i in xrange(nbfiles)]
listfiles = []
for file in files:
listfiles.append({
'scope': scope,
'name': file,
'size': filesize
})
for source_rse in source_rses:
add_file_replica(source_rse,
scope,
file,
filesize,
issuer=account)
stime3 = time.time()
print 'Time to create replicas : %s' % str(stime3 - stime2)
monitor.record(timeseries='dbfiller.addreplicas',
delta=nbfiles * len(source_rses))
attach_identifier(scope,
name=dsn,
dids=listfiles,
issuer=account)
stime4 = time.time()
print 'Time to attach files : %s' % str(stime4 - stime3)
monitor.record(timeseries='dbfiller.addnewfile',
delta=nbfiles)
for source_rse in source_rses:
try:
add_replication_rule(dids=[{
'scope': scope,
'name': dsn
}],
account=account,
copies=1,
rse_expression=source_rse,
grouping='DATASET',
weight=None,
lifetime=None,
locked=False,
subscription_id=None,
issuer='root')
monitor.record(
timeseries='dbfiller.addreplicationrules',
delta=1)
except InvalidReplicationRule, e:
print e
stime5 = time.time()
print 'Time to attach files : %s' % str(stime5 - stime4)
def POST(self):
"""
Create a new replication rule.
HTTP Success:
201 Created
HTTP Error:
400 Bad Request
401 Unauthorized
404 Not Found
409 Conflict
500 Internal Error
"""
json_data = data()
try:
grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notify,\
purge_replicas, ignore_availability, comment, ask_approval, asynchronous, priority,\
split_container, meta = 'DATASET', None, None, False, None, None, None, None, False, False, None,\
False, False, 3, False, None
params = loads(json_data)
dids = params['dids']
account = params['account']
copies = params['copies']
rse_expression = params['rse_expression']
if 'grouping' in params:
grouping = params['grouping']
if 'weight' in params:
weight = params['weight']
if 'lifetime' in params:
lifetime = params['lifetime']
if 'locked' in params:
locked = params['locked']
if 'subscription_id' in params:
subscription_id = params['subscription_id']
if 'source_replica_expression' in params:
source_replica_expression = params['source_replica_expression']
if 'activity' in params:
activity = params['activity']
if 'notify' in params:
notify = params['notify']
if 'purge_replicas' in params:
purge_replicas = params['purge_replicas']
if 'ignore_availability' in params:
ignore_availability = params['ignore_availability']
if 'comment' in params:
comment = params['comment']
if 'ask_approval' in params:
ask_approval = params['ask_approval']
if 'asynchronous' in params:
asynchronous = params['asynchronous']
if 'priority' in params:
priority = params['priority']
if 'split_container' in params:
split_container = params['split_container']
if 'meta' in params:
meta = params['meta']
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
try:
rule_ids = add_replication_rule(dids=dids,
copies=copies,
rse_expression=rse_expression,
weight=weight,
lifetime=lifetime,
grouping=grouping,
account=account,
locked=locked,
subscription_id=subscription_id,
source_replica_expression=source_replica_expression,
activity=activity,
notify=notify,
purge_replicas=purge_replicas,
ignore_availability=ignore_availability,
comment=comment,
ask_approval=ask_approval,
asynchronous=asynchronous,
priority=priority,
split_container=split_container,
meta=meta,
issuer=ctx.env.get('issuer'))
# TODO: Add all other error cases here
except InvalidReplicationRule as error:
raise generate_http_error(409, 'InvalidReplicationRule', error.args[0])
except DuplicateRule as error:
raise generate_http_error(409, 'DuplicateRule', error.args[0])
except InsufficientTargetRSEs as error:
raise generate_http_error(409, 'InsufficientTargetRSEs', error.args[0])
except InsufficientAccountLimit as error:
raise generate_http_error(409, 'InsufficientAccountLimit', error.args[0])
except InvalidRSEExpression as error:
raise generate_http_error(409, 'InvalidRSEExpression', error.args[0])
except DataIdentifierNotFound as error:
raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
except ReplicationRuleCreationTemporaryFailed as error:
raise generate_http_error(409, 'ReplicationRuleCreationTemporaryFailed', error.args[0])
except InvalidRuleWeight as error:
raise generate_http_error(409, 'InvalidRuleWeight', error.args[0])
except StagingAreaRuleRequiresLifetime as error:
raise generate_http_error(409, 'StagingAreaRuleRequiresLifetime', error.args[0])
except ScratchDiskLifetimeConflict as error:
raise generate_http_error(409, 'ScratchDiskLifetimeConflict', error.args[0])
except ManualRuleApprovalBlocked as error:
raise generate_http_error(409, 'ManualRuleApprovalBlocked', error.args[0])
except InvalidObject as error:
raise generate_http_error(409, 'InvalidObject', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__, error.args[0])
except Exception as error:
print format_exc()
raise InternalError(error)
raise Created(dumps(rule_ids))
def post(self):
"""
Create a new replication rule.
.. :quickref: AllRule; create new rule
:<json list dids: List of data identifiers.
:<json string account: Account issuing the rule.
:<json int copies: The number of replicas.
:<json string rse_expression: RSE expression which gets resolved into a list of RSEs.
:<json string grouping: ALL - All files will be replicated to the same RSE.
DATASET - All files in the same dataset will be replicated to the same RSE.
NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all
:<json int weight: Weighting scheme to be used.
:<json int lifetime: The lifetime of the replication rule in seconds.
:<json string locked: If the is locked.
:<json string subscription_id: The subscription_id, if the rule is created by a subscription.
:<json string source_replica_expression: Only use replicas as source from these RSEs.
:<json string activity: Activity to be passed to the conveyor.
:<json string notify: Notification setting of the rule ('Y', 'N', 'C'; None = 'N').
:<json bool purge_replicas: Purge setting if a replica should be directly deleted after the rule is deleted.
:<json bool ignore_availability: Option to ignore the availability of RSEs.
:<json string comments: Comment about the rule.
:<json bool ask_approval: Ask for approval for this rule.
:<json bool asynchronous: Create replication rule asynchronously by the judge-injector.
:<json int priority: Priority of the rule and the transfers which should be submitted.
:<json bool split_container: Should a container rule be split into individual dataset rules.
:<json string meta: Dictionary with metadata from the WFMS.
:status 201: rule created
:status 401: Invalid Auth Token
:status 404: DID not found
:status 409: Invalid Replication Rule
:status 409: Duplicate Replication Rule
:status 409: Insufficient Target RSEs
:status 409: Insufficient Account Limit
:status 409: Invalid RSE Expression
:status 409: Replication Rule Creation Temporary Failed
:status 409: Invalid Rule Weight
:status 409: Staging Area Rule Requires Lifetime
:status 409: Scratch Disk Lifetime Conflict
:status 409: Manual Rule Approval Blocked
:status 409: Invalid Object
:returns: List of ids for created rules
"""
json_data = request.data
try:
grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notify,\
purge_replicas, ignore_availability, comment, ask_approval, asynchronous, priority,\
split_container, meta = 'DATASET', None, None, False, None, None, None, None, False, False, None,\
False, False, 3, False, None
params = loads(json_data)
dids = params['dids']
account = params['account']
copies = params['copies']
rse_expression = params['rse_expression']
if 'grouping' in params:
grouping = params['grouping']
if 'weight' in params:
weight = params['weight']
if 'lifetime' in params:
lifetime = params['lifetime']
if 'locked' in params:
locked = params['locked']
if 'subscription_id' in params:
subscription_id = params['subscription_id']
if 'source_replica_expression' in params:
source_replica_expression = params['source_replica_expression']
if 'activity' in params:
activity = params['activity']
if 'notify' in params:
notify = params['notify']
if 'purge_replicas' in params:
purge_replicas = params['purge_replicas']
if 'ignore_availability' in params:
ignore_availability = params['ignore_availability']
if 'comment' in params:
comment = params['comment']
if 'ask_approval' in params:
ask_approval = params['ask_approval']
if 'asynchronous' in params:
asynchronous = params['asynchronous']
if 'priority' in params:
priority = params['priority']
if 'split_container' in params:
split_container = params['split_container']
if 'meta' in params:
meta = params['meta']
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
rule_ids = add_replication_rule(
dids=dids,
copies=copies,
rse_expression=rse_expression,
weight=weight,
lifetime=lifetime,
grouping=grouping,
account=account,
locked=locked,
subscription_id=subscription_id,
source_replica_expression=source_replica_expression,
activity=activity,
notify=notify,
purge_replicas=purge_replicas,
ignore_availability=ignore_availability,
comment=comment,
ask_approval=ask_approval,
asynchronous=asynchronous,
priority=priority,
split_container=split_container,
meta=meta,
issuer=request.environ.get('issuer'))
# TODO: Add all other error cases here
except InvalidReplicationRule as error:
return generate_http_error_flask(409, 'InvalidReplicationRule',
error.args[0])
except DuplicateRule as error:
return generate_http_error_flask(409, 'DuplicateRule',
error.args[0])
except InsufficientTargetRSEs as error:
return generate_http_error_flask(409, 'InsufficientTargetRSEs',
error.args[0])
except InsufficientAccountLimit as error:
return generate_http_error_flask(409, 'InsufficientAccountLimit',
error.args[0])
except InvalidRSEExpression as error:
return generate_http_error_flask(409, 'InvalidRSEExpression',
error.args[0])
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except ReplicationRuleCreationTemporaryFailed as error:
return generate_http_error_flask(
409, 'ReplicationRuleCreationTemporaryFailed', error.args[0])
except InvalidRuleWeight as error:
return generate_http_error_flask(409, 'InvalidRuleWeight',
error.args[0])
except StagingAreaRuleRequiresLifetime as error:
return generate_http_error_flask(
409, 'StagingAreaRuleRequiresLifetime', error.args[0])
except ScratchDiskLifetimeConflict as error:
return generate_http_error_flask(409,
'ScratchDiskLifetimeConflict',
error.args[0])
except ManualRuleApprovalBlocked as error:
return generate_http_error_flask(409, 'ManualRuleApprovalBlocked',
error.args[0])
except InvalidObject as error:
return generate_http_error_flask(409, 'InvalidObject',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(error)
print(format_exc())
return error, 500
return Response(dumps(rule_ids), status=201)
def post(self):
"""
Create a new replication rule.
.. :quickref: AllRule; create new rule
:<json list dids: List of data identifiers.
:<json string account: Account issuing the rule.
:<json int copies: The number of replicas.
:<json string rse_expression: RSE expression which gets resolved into a list of RSEs.
:<json string grouping: ALL - All files will be replicated to the same RSE.
DATASET - All files in the same dataset will be replicated to the same RSE.
NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all
:<json int weight: Weighting scheme to be used.
:<json int lifetime: The lifetime of the replication rule in seconds.
:<json string locked: If the is locked.
:<json string subscription_id: The subscription_id, if the rule is created by a subscription.
:<json string source_replica_expression: Only use replicas as source from these RSEs.
:<json string activity: Activity to be passed to the conveyor.
:<json string notify: Notification setting of the rule ('Y', 'N', 'C'; None = 'N').
:<json bool purge_replicas: Purge setting if a replica should be directly deleted after the rule is deleted.
:<json bool ignore_availability: Option to ignore the availability of RSEs.
:<json string comments: Comment about the rule.
:<json bool ask_approval: Ask for approval for this rule.
:<json bool asynchronous: Create replication rule asynchronously by the judge-injector.
:<json int priority: Priority of the rule and the transfers which should be submitted.
:<json bool split_container: Should a container rule be split into individual dataset rules.
:<json string meta: Dictionary with metadata from the WFMS.
:status 201: rule created
:status 401: Invalid Auth Token
:status 404: DID not found
:status 409: Invalid Replication Rule
:status 409: Duplicate Replication Rule
:status 409: Insufficient Target RSEs
:status 409: Insufficient Account Limit
:status 409: Invalid RSE Expression
:status 409: Replication Rule Creation Temporary Failed
:status 409: Invalid Rule Weight
:status 409: Staging Area Rule Requires Lifetime
:status 409: Scratch Disk Lifetime Conflict
:status 409: Manual Rule Approval Blocked
:status 409: Invalid Object
:returns: List of ids for created rules
"""
parameters = json_parameters()
dids = param_get(parameters, 'dids')
account = param_get(parameters, 'account')
copies = param_get(parameters, 'copies')
rse_expression = param_get(parameters, 'rse_expression')
try:
rule_ids = add_replication_rule(
dids=dids,
copies=copies,
rse_expression=rse_expression,
weight=param_get(parameters, 'weight', default=None),
lifetime=param_get(parameters, 'lifetime', default=None),
grouping=param_get(parameters, 'grouping', default='DATASET'),
account=account,
locked=param_get(parameters, 'locked', default=False),
subscription_id=param_get(parameters,
'subscription_id',
default=None),
source_replica_expression=param_get(
parameters, 'source_replica_expression', default=None),
activity=param_get(parameters, 'activity', default=None),
notify=param_get(parameters, 'notify', default=None),
purge_replicas=param_get(parameters,
'purge_replicas',
default=False),
ignore_availability=param_get(parameters,
'ignore_availability',
default=False),
comment=param_get(parameters, 'comment', default=None),
ask_approval=param_get(parameters,
'ask_approval',
default=False),
asynchronous=param_get(parameters,
'asynchronous',
default=False),
priority=param_get(parameters, 'priority', default=3),
split_container=param_get(parameters,
'split_container',
default=False),
meta=param_get(parameters, 'meta', default=None),
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'),
)
except (
InvalidReplicationRule,
DuplicateRule,
InsufficientTargetRSEs,
InsufficientAccountLimit,
InvalidRSEExpression,
ReplicationRuleCreationTemporaryFailed,
InvalidRuleWeight,
StagingAreaRuleRequiresLifetime,
ScratchDiskLifetimeConflict,
ManualRuleApprovalBlocked,
InvalidObject,
) as error:
return generate_http_error_flask(409, error)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
return Response(dumps(rule_ids), status=201)
def post(self):
"""
---
summary: Create a new replication rule
tags:
- Rule
requestBody:
description: Parameters for the new rule.
content:
'application/json':
schema:
type: object
required:
- dids
- account
- copies
- rse_expression
properties:
dids:
description: The list of data identifiers.
type: array
items:
type: string
account:
descipriton: The account of the issuer.
type: string
copies:
description: The number of replicas.
type: integer
rse_expression:
description: The rse expression which gets resolved into a list of RSEs.
type: string
grouping:
description: The grouping of the files to take into account. (ALL, DATASET, NONE)
type: string
weight:
description: Weighting scheme to be used.
type: number
lifetime:
description: The lifetime of the replication rule in seconds.
type: integer
locked:
description: If the rule is locked.
type: boolean
subscription_id:
description: The subscription_id, if the rule is created by a subscription.
type: string
sourse_replica_expression:
description: Only use replicas as source from these RSEs.
type: string
activity:
description: Activity to be passed to the conveyor.
type: string
notify:
description: Notification setting of the rule ('Y', 'N', 'C'; None = 'N').
type: string
purge_replicas:
description: Purge setting if a replica should be directly deleted after the rule is deleted.
type: boolean
ignore_availability:
description: Option to ignore the availability of RSEs.
type: boolean
comments:
description: Comment about the rule.
type: string
ask_approval:
description: Ask for approval for this rule.
type: boolean
asynchronous:
description: Create replication rule asynchronously by the judge-injector.
type: boolean
priority:
description: Priority of the rule and the transfers which should be submitted.
type: integer
split_container:
description: Should a container rule be split into individual dataset rules.
type: boolean
meta:
description: Dictionary with metadata from the WFMS.
type: string
responses:
201:
description: Rule created.
content:
application/json:
schema:
type: array
items:
type: string
description: Id of each created rule.
401:
description: Invalid Auth Token
404:
description: No rule found for the given id
409:
description: |
- Invalid Replication Rule
- Duplicate Replication Rule
- Insufficient Target RSEs
- Insufficient Account Limit
- Invalid RSE Expression
- Replication Rule Creation Temporary Failed,
- Invalid Rule Weight
- Staging Area Rule Requires Lifetime
- Scratch Disk Lifetime Conflict
- Manual Rule Approval Blocked
- Invalid Object
"""
parameters = json_parameters()
dids = param_get(parameters, 'dids')
account = param_get(parameters, 'account')
copies = param_get(parameters, 'copies')
rse_expression = param_get(parameters, 'rse_expression')
try:
rule_ids = add_replication_rule(
dids=dids,
copies=copies,
rse_expression=rse_expression,
weight=param_get(parameters, 'weight', default=None),
lifetime=param_get(parameters, 'lifetime', default=None),
grouping=param_get(parameters, 'grouping', default='DATASET'),
account=account,
locked=param_get(parameters, 'locked', default=False),
subscription_id=param_get(parameters,
'subscription_id',
default=None),
source_replica_expression=param_get(
parameters, 'source_replica_expression', default=None),
activity=param_get(parameters, 'activity', default=None),
notify=param_get(parameters, 'notify', default=None),
purge_replicas=param_get(parameters,
'purge_replicas',
default=False),
ignore_availability=param_get(parameters,
'ignore_availability',
default=False),
comment=param_get(parameters, 'comment', default=None),
ask_approval=param_get(parameters,
'ask_approval',
default=False),
asynchronous=param_get(parameters,
'asynchronous',
default=False),
delay_injection=param_get(parameters,
'delay_injection',
default=None),
priority=param_get(parameters, 'priority', default=3),
split_container=param_get(parameters,
'split_container',
default=False),
meta=param_get(parameters, 'meta', default=None),
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'),
)
except (
InvalidReplicationRule,
DuplicateRule,
InsufficientTargetRSEs,
InsufficientAccountLimit,
InvalidRSEExpression,
ReplicationRuleCreationTemporaryFailed,
InvalidRuleWeight,
StagingAreaRuleRequiresLifetime,
ScratchDiskLifetimeConflict,
ManualRuleApprovalBlocked,
InvalidObject,
) as error:
return generate_http_error_flask(409, error)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
return Response(dumps(rule_ids), status=201)
def populateDB(filename=None):
listrses = list_rses(filters={'deterministic': 1})
print listrses
listrses = map(lambda x: x['rse'], listrses)
account = 'root'
nbDatasets = 0
list = []
dictDistrib = {}
if not filename:
if os.getenv('RUCIO_HOME'):
filename = os.getenv('RUCIO_HOME') + '/etc/data12_8TeV_distribution.txt'
else:
filename = '/opt/rucio/etc/data12_8TeV_distribution.txt'
# Get the dataset distribution
f = open(filename, 'r')
for line in f:
if not line.startswith('NBDATASETS'):
line = line.rstrip('\n')
strsplit = line.split()
dictDistrib[(nbDatasets, nbDatasets + int(strsplit[0]))] = strsplit[1:]
nbDatasets += int(strsplit[0])
list.append([nbDatasets, ] + strsplit[1:])
f.close()
# Generate 200000 datasets according to the dataset distribution
for i in xrange(0, 200000):
rnd = random.random() * nbDatasets
for lower, upper in dictDistrib:
if (rnd > lower) and (rnd < upper):
project = dictDistrib[lower, upper][0]
scope = project
run_number = random.randint(0, 1000000)
tag = random.randint(0, 10000)
stream_name = dictDistrib[lower, upper][1]
prod_step = dictDistrib[lower, upper][2]
datatype = dictDistrib[lower, upper][3]
provenance = dictDistrib[lower, upper][4]
group = dictDistrib[lower, upper][5]
if group == '/atlas/role=production':
# account = 'atlasprod'
account = 'panda'
if provenance == 'T0':
group = 'tier0'
account = 'tier0'
else:
group = 'panda'
else:
# account = dictGroups[group]
account = 'panda'
scope = 'group.%s' % (dictGroups[group])
group = dictGroups[group]
nbfiles = int(dictDistrib[lower, upper][6])
filesize = int(int(dictDistrib[lower, upper][7])/float(nbfiles))
nbreplicas = int(dictDistrib[lower, upper][8])
if group == 'panda' or group == 'tier0':
dataset_meta = {'project': project, 'run_number': run_number, 'stream_name': stream_name, 'prod_step': prod_step, 'datatype': datatype, 'provenance': provenance, 'group': group}
else:
campaign = int(tag/1000.)
dataset_meta = {'project': project, 'run_number': run_number, 'stream_name': stream_name, 'prod_step': prod_step, 'datatype': datatype, 'provenance': provenance, 'group': group, 'campaign': '%s_repro_%i' % (group, campaign)}
source_rses = []
if nbreplicas:
iter = 0
while (len(source_rses) != nbreplicas and iter != 100):
rnd_site = random.choice(listrses)
iter += 1
if (rnd_site not in source_rses):
source_rses.append(rnd_site)
run_number_string = str(run_number)
run_number_string = run_number_string.rjust(7, '0')
dsn = '%s.%s.%s.%s.%s.%s' % (project, run_number_string, stream_name, prod_step, datatype, tag)
print '%i Creating %s:%s with %i files of size %i located at %i sites' % (i, scope, dsn, nbfiles, filesize, len(source_rses))
stime1 = time.time()
add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True}, meta=dataset_meta)
stime2 = time.time()
print 'Time to generate a dataset : %s' % str(stime2 - stime1)
monitor.record(timeseries='dbfiller.addnewdataset', delta=1)
files = ['file_%s' % uuid() for i in xrange(nbfiles)]
listfiles = []
for file in files:
listfiles.append({'scope': scope, 'name': file, 'size': filesize})
for source_rse in source_rses:
add_file_replica(source_rse, scope, file, filesize, issuer=account)
stime3 = time.time()
print 'Time to create replicas : %s' % str(stime3 - stime2)
monitor.record(timeseries='dbfiller.addreplicas', delta=nbfiles*len(source_rses))
attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
stime4 = time.time()
print 'Time to attach files : %s' % str(stime4 - stime3)
monitor.record(timeseries='dbfiller.addnewfile', delta=nbfiles)
for source_rse in source_rses:
try:
add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=source_rse,
grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer='root')
monitor.record(timeseries='dbfiller.addreplicationrules', delta=1)
except InvalidReplicationRule, e:
print e
stime5 = time.time()
print 'Time to attach files : %s' % str(stime5 - stime4)
def POST(self):
"""
Create a new replication rule.
HTTP Success:
201 Created
HTTP Error:
400 Bad Request
401 Unauthorized
404 Not Found
409 Conflict
500 Internal Error
"""
json_data = data()
try:
grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notify, purge_replicas, ignore_availability = 'DATASET', None, None, False, None, None, None, None, False, False
params = loads(json_data)
dids = params['dids']
account = params['account']
copies = params['copies']
rse_expression = params['rse_expression']
if 'grouping' in params:
grouping = params['grouping']
if 'weight' in params:
weight = params['weight']
if 'lifetime' in params:
lifetime = params['lifetime']
if 'locked' in params:
locked = params['locked']
if 'subscription_id' in params:
subscription_id = params['subscription_id']
if 'source_replica_expression' in params:
source_replica_expression = params['source_replica_expression']
if 'activity' in params:
activity = params['activity']
if 'notify' in params:
notify = params['notify']
if 'purge_replicas' in params:
purge_replicas = params['purge_replicas']
if 'ignore_availability' in params:
ignore_availability = params['ignore_availability']
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
try:
rule_ids = add_replication_rule(dids=dids,
copies=copies,
rse_expression=rse_expression,
weight=weight,
lifetime=lifetime,
grouping=grouping,
account=account,
locked=locked,
subscription_id=subscription_id,
source_replica_expression=source_replica_expression,
activity=activity,
notify=notify,
purge_replicas=purge_replicas,
ignore_availability=ignore_availability,
issuer=ctx.env.get('issuer'))
# TODO: Add all other error cases here
except InvalidReplicationRule, e:
raise generate_http_error(409, 'InvalidReplicationRule', e.args[0][0])
