def test_update_replicas_paths(self):
""" REPLICA (CORE): Force update the replica path """
tmp_scope = 'mock'
nbfiles = 5
rse_info = rsemgr.get_rse_info('MOCK')
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'pfn':
'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/does/not/really/matter/where',
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
},
'rse_id': rse_info['id'],
'path': '/does/not/really/matter/where'
} for _ in range(nbfiles)]
add_replicas(rse='MOCK2',
files=files,
account='root',
ignore_availability=True)
update_replicas_paths(files)
for replica in list_replicas(dids=[{
'scope': f['scope'],
'name': f['name'],
'type': DIDType.FILE
} for f in files],
schemes=['srm']):
# force the changed string - if we look it up from the DB, then we're not testing anything :-D
assert_equal(
replica['rses']['MOCK2'][0],
'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/does/not/really/matter/where'
)
def test_update_replicas_paths(self):
""" REPLICA (CORE): Force update the replica path """
tmp_scope = 'mock'
nbfiles = 5
rse_info = rsemgr.get_rse_info('MOCK')
files = [{'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'pfn': 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests//does/not/really/matter/where',
'bytes': 1L,
'adler32': '0cc737eb',
'meta': {'events': 10},
'rse_id': rse_info['id'],
'path': '/does/not/really/matter/where'} for i in xrange(nbfiles)]
add_replicas(rse='MOCK2', files=files, account='root', ignore_availability=True)
update_replicas_paths(files)
for replica in list_replicas(dids=[{'scope': f['scope'],
'name': f['name'],
'type': DIDType.FILE} for f in files],
schemes=['srm']):
# force the changed string - if we look it up from the DB, then we're not testing anything :-D
assert_equal(replica['rses']['MOCK2'][0], 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests//does/not/really/matter/where')
def test_add_list_bad_replicas(self):
""" REPLICA (CORE): Add bad replicas and list them"""
tmp_scope = 'mock'
nbfiles = 5
# Adding replicas to deterministic RSE
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in xrange(nbfiles)]
rse_info = rsemgr.get_rse_info('MOCK')
rse_id1 = rse_info['id']
'name': 'file_%s' % generate_uuid(),
'pfn':
'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/does/not/really/matter/where',
'bytes': 1L,
'adler32': '0cc737eb',
'meta': {
'events': 10
},
'rse_id': rse_info['id'],
'path': '/does/not/really/matter/where'
} for i in xrange(nbfiles)]
add_replicas(rse='MOCK2',
files=files,
account='root',
ignore_availability=True)
update_replicas_paths(files)
for replica in list_replicas(dids=[{
'scope': f['scope'],
'name': f['name'],
'type': DIDType.FILE
} for f in files],
schemes=['srm']):
# force the changed string - if we look it up from the DB, then we're not testing anything :-D
assert_equal(
replica['rses']['MOCK2'][0],
'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/does/not/really/matter/where'
)
def test_add_list_bad_replicas(self):
""" REPLICA (CORE): Add bad replicas and list them"""
tmp_scope = 'mock'
def get_destinations(rse_info, scheme, req, sources):
dsn = 'other'
pfn = {}
paths = {}
if not rse_info['deterministic']:
ts = time.time()
# get rule scope and name
if req['attributes']:
if type(req['attributes']) is dict:
req_attributes = json.loads(json.dumps(req['attributes']))
else:
req_attributes = json.loads(str(req['attributes']))
if 'ds_name' in req_attributes:
dsn = req_attributes["ds_name"]
if dsn == 'other':
# select a containing dataset
for parent in did.list_parent_dids(req['scope'], req['name']):
if parent['type'] == DIDType.DATASET:
dsn = parent['name']
break
record_timer('daemons.conveyor.submitter.list_parent_dids', (time.time() - ts) * 1000)
# always use SRM
ts = time.time()
nondet = rsemgr.create_protocol(rse_info, 'write', scheme='srm')
record_timer('daemons.conveyor.submitter.create_protocol', (time.time() - ts) * 1000)
# if there exists a prefix for SRM, use it
prefix = ''
for s in rse_info['protocols']:
if s['scheme'] == 'srm':
prefix = s['prefix']
# DQ2 path always starts with /, but prefix might not end with /
path = construct_surl_DQ2(dsn, req['name'])
# retrial transfers to tape need a new filename - add timestamp
if req['request_type'] == RequestType.TRANSFER\
and 'previous_attempt_id' in req\
and req['previous_attempt_id']\
and rse_info['rse_type'] == 'TAPE': # TODO: RUCIO-809 - rsemanager: get_rse_info -> rse_type is string instead of RSEType
path = '%s_%i' % (path, int(time.time()))
logging.debug('Retrial transfer request %s DID %s:%s to tape %s renamed to %s' % (req['request_id'],
req['scope'],
req['name'],
rse_info['rse'],
path))
tmp_path = '%s%s' % (prefix[:-1], path)
if prefix[-1] != '/':
tmp_path = '%s%s' % (prefix, path)
paths[req['scope'], req['name']] = path
# add the hostname
pfn['%s:%s' % (req['scope'], req['name'])] = nondet.path2pfn(tmp_path)
if req['request_type'] == RequestType.STAGEIN:
if len(sources) == 1:
pfn['%s:%s' % (req['scope'], req['name'])] = sources[0][1]
else:
# TODO: need to check
return None, None
# we must set the destination path for nondeterministic replicas explicitly
replica.update_replicas_paths([{'scope': req['scope'],
'name': req['name'],
'rse_id': req['dest_rse_id'],
'path': path}])
else:
ts = time.time()
try:
pfn = rsemgr.lfns2pfns(rse_info,
lfns=[{'scope': req['scope'],
'name': req['name']}],
scheme=scheme)
except RSEProtocolNotSupported:
logging.warn('%s not supported by %s' % (scheme, rse_info['rse']))
return None, None
record_timer('daemons.conveyor.submitter.lfns2pfns', (time.time() - ts) * 1000)
destinations = []
for k in pfn:
if isinstance(pfn[k], (str, unicode)):
destinations.append(pfn[k])
elif isinstance(pfn[k], (tuple, list)):
for url in pfn[k]:
destinations.append(pfn[k][url])
protocols = None
try:
protocols = rsemgr.select_protocol(rse_info, 'write', scheme=scheme)
except RSEProtocolNotSupported:
logging.warn('%s not supported by %s' % (scheme, rse_info['rse']))
return None, None
# we need to set the spacetoken if we use SRM
dest_spacetoken = None
if scheme == 'srm':
dest_spacetoken = protocols['extended_attributes']['space_token']
return destinations, dest_spacetoken
