def get(self):
""" List all RSEs.
.. :quickref: RSEs; List all RSEs.
:query expression: The returned list only contains RSE matching this expression.
:resheader Content-Type: application/x-json-stream
:status 200: DIDs found.
:status 400: Invalid RSE Expression.
:status 401: Invalid Auth Token.
:status 500: Internal Error.
:returns: A list containing all RSEs.
"""
expression = request.args.get('name', None)
if expression:
try:
data = ""
for rse in parse_rse_expression(expression):
item = {'rse': rse}
data += render_json(**item) + '\n'
return Response(data, content_type="application/x-json-stream")
except InvalidRSEExpression as error:
return generate_http_error_flask(400, 'InvalidRSEExpression', error.args[0])
except InvalidObject as error:
return generate_http_error_flask(400, 'InvalidObject', error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
else:
data = ""
for rse in list_rses():
data += render_json(**rse) + '\n'
return Response(data, content_type="application/x-json-stream")
def GET(self):
""" List all RSEs.
HTTP Success:
200 OK
HTTP Error:
400 Bad request
401 Unauthorized
404 Resource not Found
406 Not Acceptable
500 InternalError
:returns: A list containing all RSEs.
"""
header('Content-Type', 'application/x-json-stream')
params = input()
if 'expression' in params:
try:
for rse in parse_rse_expression(params['expression'], vo=ctx.env.get('vo')):
item = {'rse': rse}
yield render_json(**item) + '\n'
except InvalidRSEExpression as error:
raise generate_http_error(400, 'InvalidRSEExpression', error.args[0])
except InvalidObject as error:
raise generate_http_error(400, 'InvalidObject', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__, error.args[0])
else:
for rse in list_rses(vo=ctx.env.get('vo')):
yield render_json(**rse) + '\n'
def test_rses_at_different_vos(self):
""" MULTI VO (CLIENT): Test that RSEs from 2nd vo don't interfere """
# Set up RSEs at two VOs
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst = 'TST_%s' % rse_str
new = 'NEW_%s' % rse_str
shr = 'SHR_%s' % rse_str
rse_client.add_rse(tst)
rse_client.add_rse(shr)
add_rse(new, 'root', **self.new_vo)
shr_id_new_original = add_rse(shr, 'root', **self.new_vo) # Accurate rse_id for shared RSE at 'new'
# Check the cached rse-id from each VO does not interfere
shr_id_tst = get_rse_id(shr, **self.vo)
shr_id_new = get_rse_id(shr, **self.new_vo)
assert_equal(shr_id_new, shr_id_new_original)
assert_not_equal(shr_id_new, shr_id_tst)
# Check that when listing RSEs we only get RSEs for our VO
rse_list_tst = [r['rse'] for r in rse_client.list_rses()]
rse_list_new = [r['rse'] for r in list_rses(filters={}, **self.new_vo)]
assert_true(tst in rse_list_tst)
assert_false(new in rse_list_tst)
assert_true(shr in rse_list_tst)
assert_false(tst in rse_list_new)
assert_true(new in rse_list_new)
assert_true(shr in rse_list_new)
# Check the cached attribute-value results do not interfere and only give results from the appropriate VO
attribute_value = generate_uuid()
add_rse_attribute(new, 'test', attribute_value, 'root', **self.new_vo)
rses_tst_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo))
rses_new_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo))
rses_tst_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo))
rses_new_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo))
assert_equal(len(rses_tst_1), 0)
assert_not_equal(len(rses_new_1), 0)
assert_equal(len(rses_tst_2), 0)
assert_not_equal(len(rses_new_2), 0)
# check parse_expression
rses_tst_3 = parse_expression(shr, filter={'vo': self.vo['vo']})
rses_tst_4 = parse_expression(tst, filter={'vo': self.vo['vo']})
rses_new_3 = parse_expression(shr, filter={'vo': self.new_vo['vo']})
with assert_raises(InvalidRSEExpression):
parse_expression(tst, filter={'vo': self.new_vo['vo']})
assert_equal(len(rses_tst_3), 1)
assert_equal(shr_id_tst, rses_tst_3[0]['id'])
assert_equal(len(rses_tst_4), 1)
assert_equal(tst, rses_tst_4[0]['rse'])
assert_equal(len(rses_new_3), 1)
assert_equal(shr_id_new, rses_new_3[0]['id'])
def populateDB():
listrses = list_rses({'T1': '1'})
print len(listrses), listrses
# listrses = list_rses()
# print len(listrses), listrses
# sys.exit()
account = 'root'
project = 'mc12_8TeV'
dictDistrib = [{'datatype': 'HITS', 'prodstep': 'merge', 'nbfiles': 302, 'totfilesize': 225394185112, 'nbreplicas': 1}, {'datatype': 'HITS', 'prodstep': 'simul', 'nbfiles': 620, 'totfilesize': 97930909866, 'nbreplicas': 1},
{'datatype': 'EVNT', 'prodstep': 'evgen', 'nbfiles': 324, 'totfilesize': 7809298802, 'nbreplicas': 3}, {'datatype': 'AOD', 'prodstep': 'merge', 'nbfiles': 52, 'totfilesize': 106942334943, 'nbreplicas': 4},
{'datatype': 'AOD', 'prodstep': 'recon', 'nbfiles': 858, 'totfilesize': 182186965627, 'nbreplicas': 1}]
for d in dictDistrib:
for day in xrange(0, 180):
for i in xrange(0, 30):
scope = project
prod_step = d['prodstep']
datatype = d['datatype']
nbfiles = int(d['nbfiles'])
filesize = int(int(d['totfilesize'])/float(nbfiles))
nbfiles = int(random.gauss(nbfiles, nbfiles/10))
filesize = int(random.gauss(filesize, filesize/10))
nbreplicas = int(d['nbreplicas'])
dataset_meta = {'project': project, 'stream_name': 'dummy', 'prod_step': prod_step, 'datatype': datatype}
source_rses = []
if nbreplicas:
iter = 0
while (len(source_rses) != nbreplicas and iter != 100):
rnd_site = random.choice(listrses)
iter += 1
if rnd_site not in source_rses:
source_rses.append(rnd_site)
try:
dsn = '%s.%s.%s.%i.%i' % (project, prod_step, datatype, day, i)
print '%i Creating %s with %i files of size %i located at %i sites' % (i, dsn, nbfiles, filesize, len(source_rses))
add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True}, meta=dataset_meta)
files = ['file_%s' % uuid() for i in xrange(nbfiles)]
listfiles = []
for file in files:
listfiles.append({'scope': scope, 'name': file, 'size': filesize})
for source_rse in source_rses:
add_file_replica(source_rse, scope, file, filesize, issuer=account)
attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
for source_rse in source_rses:
try:
add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=source_rse,
grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer='root')
except InvalidReplicationRule, e:
print e
except RucioException, e:
print e
def populateDB():
listrses = list_rses({'T1': '1'})
print(len(listrses), listrses)
# listrses = list_rses()
# print len(listrses), listrses
# sys.exit()
account = 'root'
project = 'mc12_8TeV'
dictDistrib = [{'datatype': 'HITS', 'prodstep': 'merge', 'nbfiles': 302, 'totfilesize': 225394185112, 'nbreplicas': 1}, {'datatype': 'HITS', 'prodstep': 'simul', 'nbfiles': 620, 'totfilesize': 97930909866, 'nbreplicas': 1},
{'datatype': 'EVNT', 'prodstep': 'evgen', 'nbfiles': 324, 'totfilesize': 7809298802, 'nbreplicas': 3}, {'datatype': 'AOD', 'prodstep': 'merge', 'nbfiles': 52, 'totfilesize': 106942334943, 'nbreplicas': 4},
{'datatype': 'AOD', 'prodstep': 'recon', 'nbfiles': 858, 'totfilesize': 182186965627, 'nbreplicas': 1}]
for d in dictDistrib:
for day in range(180):
for i in range(30):
scope = project
prod_step = d['prodstep']
datatype = d['datatype']
nbfiles = int(d['nbfiles'])
filesize = int(int(d['totfilesize']) / float(nbfiles))
nbfiles = int(random.gauss(nbfiles, nbfiles / 10))
filesize = int(random.gauss(filesize, filesize / 10))
nbreplicas = int(d['nbreplicas'])
dataset_meta = {'project': project, 'stream_name': 'dummy', 'prod_step': prod_step, 'datatype': datatype}
source_rses = []
if nbreplicas:
iter = 0
while (len(source_rses) != nbreplicas and iter != 100):
rnd_site = random.choice(listrses)
iter += 1
if rnd_site not in source_rses:
source_rses.append(rnd_site)
try:
dsn = '%s.%s.%s.%i.%i' % (project, prod_step, datatype, day, i)
print('%i Creating %s with %i files of size %i located at %i sites' % (i, dsn, nbfiles, filesize, len(source_rses)))
add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True}, meta=dataset_meta)
files = ['file_%s' % uuid() for i in range(nbfiles)]
listfiles = []
for file in files:
listfiles.append({'scope': scope, 'name': file, 'size': filesize})
for source_rse in source_rses:
add_file_replica(source_rse, scope, file, filesize, issuer=account)
attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
for source_rse in source_rses:
try:
add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=source_rse,
grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer='root')
except InvalidReplicationRule as e:
print(e)
except RucioException as e:
print(e)
def populateDB(filename=None):
listrses = list_rses(filters={'deterministic': 1})
listrses = map(lambda x: x['rse'], listrses)
account = 'root'
pdf = generatePDF()
# Generate 200000 datasets according to the dataset distribution
for index in range(20000):
scope_nb = getRandomScope(pdf)
project = 'user.user%i' % (scope_nb)
scope = 'user.user%i' % (scope_nb)
account = 'user%i' % (scope_nb)
print(scope)
nbfiles = 53
filesize = 78000000
uid = uuid()
dsn = '%s.%s' % (project, uid)
rnd_site = random.choice(listrses)
print('%i Creating %s with %i files of size %i located at %s' %
(index, dsn, nbfiles, filesize, rnd_site))
add_identifier(scope=scope,
name=dsn,
type='dataset',
issuer=account,
statuses={'monotonic': True})
monitor.record(timeseries='dbfiller.addnewdataset', delta=1)
files = ['file_%s' % uuid() for i in range(nbfiles)]
listfiles = []
for file in files:
listfiles.append({'scope': scope, 'name': file, 'size': filesize})
add_file_replica(rnd_site, scope, file, filesize, issuer=account)
monitor.record(timeseries='dbfiller.addreplicas', delta=nbfiles)
attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
monitor.record(timeseries='dbfiller.addnewfile', delta=nbfiles)
try:
add_replication_rule(dids=[{
'scope': scope,
'name': dsn
}],
account=account,
copies=1,
rse_expression=rnd_site,
grouping='DATASET',
weight=None,
lifetime=None,
locked=False,
subscription_id=None,
issuer=account)
monitor.record(timeseries='dbfiller.addreplicationrules', delta=1)
except InvalidReplicationRule as e:
print(e)
def __init__(self,
block_name,
pnn,
rse=None,
lifetime=None,
dry_run=False):
"""
Get the status of replica of pditem at pnn
considering only closed blocks completely replicated at site.
:rds: PhEDEx block name.
:pnn: PhEDEx node name.
:rse: Rucio RSE. If None (default) inferred by the pnn using DEFAULT_RSE_FMT.
:scope: Scope. Default: DEFAULT_SCOPE.
"""
self.phedex_svc = PhEDEx()
self.dry_run = dry_run
self.pnn = pnn
if rse is None:
self.rse = list_rses('cms_type=real&pnn=%s' % self.pnn)[0]['rse']
else:
self.rse = rse
rse_details = get_rse(self.rse)
self.rse_id = rse_details['id']
self.account = (SYNC_ACCOUNT_FMT % self.rse.lower())[:25]
self.container = self.phedex_svc.check_data_item(
pditem=block_name)['pds']
self.scope = DEFAULT_SCOPE
self.block_name = block_name
self.lifetime = lifetime
self.group, self.custodial, self.is_at_pnn = self.phedex_svc.block_at_pnn_phedex(
block=self.block_name, pnn=self.pnn)
self.block_in_phedex = self.phedex_svc.block_exists(
block=self.block_name)
self.block_known = self.phedex_svc.block_known(block=self.block_name)
if self.is_at_pnn:
self.replicas = self.phedex_svc.fileblock_files_phedex(
pnn=pnn, pfb=block_name)
else:
self.replicas = {}
self.container_exists = None
self.block_exists = None
self.rule_exists = None
touch(text=self.rse)
def populateDB(filename=None):
listrses = list_rses(filters={'deterministic': 1})
listrses = map(lambda x: x['rse'], listrses)
account = 'root'
pdf = generatePDF()
# Generate 200000 datasets according to the dataset distribution
for index in xrange(0, 20000):
scope_nb = getRandomScope(pdf)
project = 'user.user%i' % (scope_nb)
scope = 'user.user%i' % (scope_nb)
account = 'user%i' % (scope_nb)
print scope
nbfiles = 53
filesize = 78000000
uid = uuid()
dsn = '%s.%s' % (project, uid)
rnd_site = random.choice(listrses)
print '%i Creating %s with %i files of size %i located at %s' % (index, dsn, nbfiles, filesize, rnd_site)
add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True})
monitor.record(timeseries='dbfiller.addnewdataset', delta=1)
files = ['file_%s' % uuid() for i in xrange(nbfiles)]
listfiles = []
for file in files:
listfiles.append({'scope': scope, 'name': file, 'size': filesize})
add_file_replica(rnd_site, scope, file, filesize, issuer=account)
monitor.record(timeseries='dbfiller.addreplicas', delta=nbfiles)
attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
monitor.record(timeseries='dbfiller.addnewfile', delta=nbfiles)
try:
add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=rnd_site,
grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer=account)
monitor.record(timeseries='dbfiller.addreplicationrules', delta=1)
except InvalidReplicationRule, e:
print e
def populateDB(filename=None):
listrses = list_rses(filters={'deterministic': 1})
print listrses
listrses = map(lambda x: x['rse'], listrses)
account = 'root'
nbDatasets = 0
list = []
dictDistrib = {}
if not filename:
if os.getenv('RUCIO_HOME'):
filename = os.getenv(
'RUCIO_HOME') + '/etc/data12_8TeV_distribution.txt'
else:
filename = '/opt/rucio/etc/data12_8TeV_distribution.txt'
# Get the dataset distribution
f = open(filename, 'r')
for line in f:
if not line.startswith('NBDATASETS'):
line = line.rstrip('\n')
strsplit = line.split()
dictDistrib[(nbDatasets,
nbDatasets + int(strsplit[0]))] = strsplit[1:]
nbDatasets += int(strsplit[0])
list.append([
nbDatasets,
] + strsplit[1:])
f.close()
# Generate 200000 datasets according to the dataset distribution
for i in xrange(0, 200000):
rnd = random.random() * nbDatasets
for lower, upper in dictDistrib:
if (rnd > lower) and (rnd < upper):
project = dictDistrib[lower, upper][0]
scope = project
run_number = random.randint(0, 1000000)
tag = random.randint(0, 10000)
stream_name = dictDistrib[lower, upper][1]
prod_step = dictDistrib[lower, upper][2]
datatype = dictDistrib[lower, upper][3]
provenance = dictDistrib[lower, upper][4]
group = dictDistrib[lower, upper][5]
if group == '/atlas/role=production':
# account = 'atlasprod'
account = 'panda'
if provenance == 'T0':
group = 'tier0'
account = 'tier0'
else:
group = 'panda'
else:
# account = dictGroups[group]
account = 'panda'
scope = 'group.%s' % (dictGroups[group])
group = dictGroups[group]
nbfiles = int(dictDistrib[lower, upper][6])
filesize = int(
int(dictDistrib[lower, upper][7]) / float(nbfiles))
nbreplicas = int(dictDistrib[lower, upper][8])
if group == 'panda' or group == 'tier0':
dataset_meta = {
'project': project,
'run_number': run_number,
'stream_name': stream_name,
'prod_step': prod_step,
'datatype': datatype,
'provenance': provenance,
'group': group
}
else:
campaign = int(tag / 1000.)
dataset_meta = {
'project': project,
'run_number': run_number,
'stream_name': stream_name,
'prod_step': prod_step,
'datatype': datatype,
'provenance': provenance,
'group': group,
'campaign': '%s_repro_%i' % (group, campaign)
}
source_rses = []
if nbreplicas:
iter = 0
while (len(source_rses) != nbreplicas and iter != 100):
rnd_site = random.choice(listrses)
iter += 1
if (rnd_site not in source_rses):
source_rses.append(rnd_site)
run_number_string = str(run_number)
run_number_string = run_number_string.rjust(7, '0')
dsn = '%s.%s.%s.%s.%s.%s' % (project, run_number_string,
stream_name, prod_step,
datatype, tag)
print '%i Creating %s:%s with %i files of size %i located at %i sites' % (
i, scope, dsn, nbfiles, filesize, len(source_rses))
stime1 = time.time()
add_identifier(scope=scope,
name=dsn,
type='dataset',
issuer=account,
statuses={'monotonic': True},
meta=dataset_meta)
stime2 = time.time()
print 'Time to generate a dataset : %s' % str(stime2 -
stime1)
monitor.record(timeseries='dbfiller.addnewdataset',
delta=1)
files = ['file_%s' % uuid() for i in xrange(nbfiles)]
listfiles = []
for file in files:
listfiles.append({
'scope': scope,
'name': file,
'size': filesize
})
for source_rse in source_rses:
add_file_replica(source_rse,
scope,
file,
filesize,
issuer=account)
stime3 = time.time()
print 'Time to create replicas : %s' % str(stime3 - stime2)
monitor.record(timeseries='dbfiller.addreplicas',
delta=nbfiles * len(source_rses))
attach_identifier(scope,
name=dsn,
dids=listfiles,
issuer=account)
stime4 = time.time()
print 'Time to attach files : %s' % str(stime4 - stime3)
monitor.record(timeseries='dbfiller.addnewfile',
delta=nbfiles)
for source_rse in source_rses:
try:
add_replication_rule(dids=[{
'scope': scope,
'name': dsn
}],
account=account,
copies=1,
rse_expression=source_rse,
grouping='DATASET',
weight=None,
lifetime=None,
locked=False,
subscription_id=None,
issuer='root')
monitor.record(
timeseries='dbfiller.addreplicationrules',
delta=1)
except InvalidReplicationRule, e:
print e
stime5 = time.time()
print 'Time to attach files : %s' % str(stime5 - stime4)
def generate(vo):
for rse in list_rses(vo=vo):
yield render_json(**rse) + '\n'
params = input()
if 'expression' in params:
try:
for rse in parse_rse_expression(params['expression']):
item = {'rse': rse}
yield render_json(**item) + '\n'
except InvalidRSEExpression, error:
raise generate_http_error(400, 'InvalidRSEExpression',
error[0][0])
except InvalidObject, error:
raise generate_http_error(400, 'InvalidObject', error[0][0])
except RucioException, error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0][0])
else:
for rse in list_rses():
yield render_json(**rse) + '\n'
class RSE(RucioController):
""" Create, update, get and disable RSE. """
def POST(self, rse):
""" Create RSE with given name.
HTTP Success:
201 Created
HTTP Error:
400 Bad request
401 Unauthorized
404 Resource not Found
def test_api_rse(self):
""" RSE (API): Test external representation of RSEs """
out = api_rse.get_rse(self.rse_name, **self.vo)
assert out['rse'] == self.rse_name
assert out['id'] == self.rse_id
out = api_rse.list_rses(**self.new_vo)
out = list(out)
assert 0 != len(out)
rse_ids = [rse['id'] for rse in out]
assert self.rse3_id in rse_ids
assert self.rse4_id in rse_ids
for rse in out:
assert 'rse' in rse
if rse['id'] == self.rse3_id:
assert rse['rse'] == self.rse3_name
elif rse['id'] == self.rse4_id:
assert rse['rse'] == self.rse4_name
key = "KEY_" + generate_uuid()
api_rse.add_rse_attribute(self.rse_name,
key,
1,
issuer='root',
**self.vo)
out = api_rse.get_rses_with_attribute(key)
out = list(out)
assert 0 != len(out)
for rse in out:
assert rse['rse'] == self.rse_name
out = api_rse.get_rse_protocols(self.rse_name,
issuer='root',
**self.vo)
assert out['rse'] == self.rse_name
# add some account and RSE counters
rse_mock = 'MOCK4'
rse_mock_id = get_rse_id(rse_mock, **self.vo)
account_counter.del_counter(rse_id=rse_mock_id, account=self.account)
account_counter.add_counter(rse_id=rse_mock_id, account=self.account)
account_counter.increase(rse_id=rse_mock_id,
account=self.account,
files=1,
bytes=10)
account_counter.update_account_counter(self.account, rse_mock_id)
did = 'file_' + generate_uuid()
add_did(self.scope_name,
did,
'DATASET',
'root',
account=self.account_name,
rse=rse_mock,
**self.vo)
abacus_rse.run(once=True)
out = api_rse.get_rse_usage(rse_mock,
per_account=True,
issuer='root',
**self.vo)
assert rse_mock_id in [o['rse_id'] for o in out]
for usage in out:
if usage['rse_id'] == rse_mock_id:
assert usage['rse'] == rse_mock
accounts = [u['account'] for u in usage['account_usages']]
assert self.account_name in accounts
if self.multi_vo:
assert self.account.internal not in accounts
# clean up files
cleaner.run(once=True)
if self.multi_vo:
reaper.run(once=True,
include_rses='vo=%s&(%s)' % (self.vo['vo'], rse_mock),
greedy=True)
else:
reaper.run(once=True, include_rses=rse_mock, greedy=True)
abacus_rse.run(once=True)
out = api_rse.parse_rse_expression(
'%s|%s' % (self.rse_name, self.rse2_name), **self.vo)
assert self.rse_name in out
assert self.rse2_name in out
assert self.rse_id not in out
assert self.rse2_id not in out
def populateDB(filename=None):
listrses = list_rses(filters={'deterministic': 1})
print listrses
listrses = map(lambda x: x['rse'], listrses)
account = 'root'
nbDatasets = 0
list = []
dictDistrib = {}
if not filename:
if os.getenv('RUCIO_HOME'):
filename = os.getenv('RUCIO_HOME') + '/etc/data12_8TeV_distribution.txt'
else:
filename = '/opt/rucio/etc/data12_8TeV_distribution.txt'
# Get the dataset distribution
f = open(filename, 'r')
for line in f:
if not line.startswith('NBDATASETS'):
line = line.rstrip('\n')
strsplit = line.split()
dictDistrib[(nbDatasets, nbDatasets + int(strsplit[0]))] = strsplit[1:]
nbDatasets += int(strsplit[0])
list.append([nbDatasets, ] + strsplit[1:])
f.close()
# Generate 200000 datasets according to the dataset distribution
for i in xrange(0, 200000):
rnd = random.random() * nbDatasets
for lower, upper in dictDistrib:
if (rnd > lower) and (rnd < upper):
project = dictDistrib[lower, upper][0]
scope = project
run_number = random.randint(0, 1000000)
tag = random.randint(0, 10000)
stream_name = dictDistrib[lower, upper][1]
prod_step = dictDistrib[lower, upper][2]
datatype = dictDistrib[lower, upper][3]
provenance = dictDistrib[lower, upper][4]
group = dictDistrib[lower, upper][5]
if group == '/atlas/role=production':
# account = 'atlasprod'
account = 'panda'
if provenance == 'T0':
group = 'tier0'
account = 'tier0'
else:
group = 'panda'
else:
# account = dictGroups[group]
account = 'panda'
scope = 'group.%s' % (dictGroups[group])
group = dictGroups[group]
nbfiles = int(dictDistrib[lower, upper][6])
filesize = int(int(dictDistrib[lower, upper][7])/float(nbfiles))
nbreplicas = int(dictDistrib[lower, upper][8])
if group == 'panda' or group == 'tier0':
dataset_meta = {'project': project, 'run_number': run_number, 'stream_name': stream_name, 'prod_step': prod_step, 'datatype': datatype, 'provenance': provenance, 'group': group}
else:
campaign = int(tag/1000.)
dataset_meta = {'project': project, 'run_number': run_number, 'stream_name': stream_name, 'prod_step': prod_step, 'datatype': datatype, 'provenance': provenance, 'group': group, 'campaign': '%s_repro_%i' % (group, campaign)}
source_rses = []
if nbreplicas:
iter = 0
while (len(source_rses) != nbreplicas and iter != 100):
rnd_site = random.choice(listrses)
iter += 1
if (rnd_site not in source_rses):
source_rses.append(rnd_site)
run_number_string = str(run_number)
run_number_string = run_number_string.rjust(7, '0')
dsn = '%s.%s.%s.%s.%s.%s' % (project, run_number_string, stream_name, prod_step, datatype, tag)
print '%i Creating %s:%s with %i files of size %i located at %i sites' % (i, scope, dsn, nbfiles, filesize, len(source_rses))
stime1 = time.time()
add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True}, meta=dataset_meta)
stime2 = time.time()
print 'Time to generate a dataset : %s' % str(stime2 - stime1)
monitor.record(timeseries='dbfiller.addnewdataset', delta=1)
files = ['file_%s' % uuid() for i in xrange(nbfiles)]
listfiles = []
for file in files:
listfiles.append({'scope': scope, 'name': file, 'size': filesize})
for source_rse in source_rses:
add_file_replica(source_rse, scope, file, filesize, issuer=account)
stime3 = time.time()
print 'Time to create replicas : %s' % str(stime3 - stime2)
monitor.record(timeseries='dbfiller.addreplicas', delta=nbfiles*len(source_rses))
attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
stime4 = time.time()
print 'Time to attach files : %s' % str(stime4 - stime3)
monitor.record(timeseries='dbfiller.addnewfile', delta=nbfiles)
for source_rse in source_rses:
try:
add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=source_rse,
grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer='root')
monitor.record(timeseries='dbfiller.addreplicationrules', delta=1)
except InvalidReplicationRule, e:
print e
stime5 = time.time()
print 'Time to attach files : %s' % str(stime5 - stime4)
"""
header('Content-Type', 'application/x-json-stream')
params = input()
if 'expression' in params:
try:
for rse in parse_rse_expression(params['expression']):
item = {'rse': rse}
yield render_json(**item) + '\n'
except InvalidRSEExpression, e:
raise generate_http_error(400, 'InvalidRSEExpression', e)
except InvalidObject, e:
raise generate_http_error(400, 'InvalidObject', e[0][0])
except RucioException, e:
raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
else:
for rse in list_rses():
yield render_json(**rse) + '\n'
class RSE(RucioController):
""" Create, update, get and disable RSE. """
def POST(self, rse):
""" Create RSE with given name.
HTTP Success:
201 Created
HTTP Error:
400 Bad request
401 Unauthorized
