def test_did_hierarchy(self):
""" DATA IDENTIFIERS (CLIENT): Check did hierarchy rule """
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(4)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(4):
self.did_client.add_did(scope, dst[i], 'DATASET', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_did(scope, cnt[i], 'CONTAINER', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1L, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1L, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': cnt[2]}, {'scope': scope, 'name': cnt[3]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
pass
def test_attach_dids_to_dids(self):
""" DATA IDENTIFIERS (CLIENT): Attach dids to dids"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
nb_datasets = 5
nb_files = 5
attachments, dsns = list(), list()
guid_to_query = None
dsn = {}
for i in xrange(nb_datasets):
attachment = {}
attachment['scope'] = tmp_scope
attachment['name'] = 'dsn.%s' % str(generate_uuid())
attachment['rse'] = tmp_rse
files = []
for i in xrange(nb_files):
files.append({'scope': tmp_scope, 'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570L, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}})
attachment['dids'] = files
guid_to_query = files[0]['meta']['guid']
dsn = {'scope': tmp_scope, 'name': attachment['name']}
dsns.append(dsn)
attachments.append(attachment)
self.did_client.add_datasets(dsns=dsns)
self.did_client.attach_dids_to_dids(attachments=attachments)
l = [i for i in self.did_client.get_dataset_by_guid(guid_to_query)]
assert_equal([dsn], l)
cnt_name = 'cnt_%s' % generate_uuid()
self.did_client.add_container(scope='mock', name=cnt_name)
with assert_raises(UnsupportedOperation):
self.did_client.attach_dids_to_dids([{'scope': 'mock', 'name': cnt_name, 'rse': tmp_rse, 'dids': attachment['dids']}])
def test_delete_replicas_from_datasets(self):
""" REPLICA (CORE): Delete replicas from dataset """
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
nbfiles = 5
files1 = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in xrange(nbfiles)]
def test_list_dids(self):
""" DATA IDENTIFIERS (CLIENT): List dids by pattern."""
tmp_scope = scope_name_generator()
tmp_files = []
tmp_files.append('file_a_1%s' % generate_uuid())
tmp_files.append('file_a_2%s' % generate_uuid())
tmp_files.append('file_b_1%s' % generate_uuid())
tmp_rse = 'MOCK'
self.scope_client.add_scope('jdoe', tmp_scope)
for tmp_file in tmp_files:
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L, '0cc737eb')
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\_a\_*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\_a\_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 1)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\__\_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='file'):
results.append(result)
assert_equal(len(results), 3)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}):
results.append(result)
assert_equal(len(results), 0)
with assert_raises(UnsupportedOperation):
self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='whateverytype')
def test_list_content(self):
""" DATA IDENTIFIERS (CLIENT): test to list contents for an identifier"""
rse = 'MOCK'
scope = 'mock'
nbfiles = 5
dataset1 = generate_uuid()
dataset2 = generate_uuid()
container = generate_uuid()
files1 = [{'scope': scope, 'name': generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb'} for i in xrange(nbfiles)]
def test_update_dids(self):
""" DATA IDENTIFIERS (CORE): Update file size and checksum"""
tmp_scope = 'mock'
dsn = 'dsn_%s' % generate_uuid()
lfn = 'lfn.%s' % str(generate_uuid())
add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root')
files = [{'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}}]
def test_add_did(self):
""" DATA IDENTIFIERS (CLIENT): Add, populate and list did content"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
tmp_dsn = 'dsn_%s' % generate_uuid()
# PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename
dataset_meta = {'project': 'data13_hip',
'run_number': 300000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m927',
}
rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}]
with assert_raises(ScopeNotFound):
self.did_client.add_dataset(scope='Nimportnawak', name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
with assert_raises(DataIdentifierNotFound):
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=[{'scope': tmp_scope, 'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()),
'bytes': 724963570L, 'adler32': '0cc737eb'}, ])
files = []
for i in xrange(5):
lfn = 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 10}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
files = []
for i in xrange(5):
lfn = '%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 100}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
self.did_client.close(scope=tmp_scope, name=tmp_dsn)
def test_list_replicas(self):
""" DATA IDENTIFIERS (CLIENT): List replicas for a container"""
rse = 'MOCK'
scope = 'mock'
dsn1 = generate_uuid()
dsn2 = generate_uuid()
cnt = generate_uuid()
files1 = []
files2 = []
for i in xrange(10):
files1.append({'scope': scope, 'name': generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb'})
def setup(self):
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.base_client = BaseClient(account='root',
ca_cert=config_get('client', 'ca_cert'),
auth_type='x509')
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.fname = generate_uuid()
rses = ['LXPLUS', 'MOCK4']
dsn = generate_uuid()
self.files = [{'scope': 'mock', 'name': self.fname, 'bytes': 1L, 'adler32': '0cc737eb'}]
def test_list_by_metadata(self):
""" DATA IDENTIFIERS (CLIENT): List did with metadata"""
dsns = []
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn1)
dataset_meta = {'project': 'data12_8TeV',
'run_number': 400000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m920',
}
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn1, meta=dataset_meta)
tmp_dsn2 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn2)
dataset_meta['run_number'] = 400001
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn2, meta=dataset_meta)
tmp_dsn3 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn3)
dataset_meta['stream_name'] = 'physics_Egamma'
dataset_meta['datatype'] = 'NTUP_SMWZ'
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn3, meta=dataset_meta)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'version': 'f392_m920'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn1)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'run_number': 400001})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn2)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'stream_name': 'physics_Egamma', 'datatype': 'NTUP_SMWZ'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
with assert_raises(KeyNotFound):
self.did_client.list_dids(tmp_scope, {'NotReallyAKey': 'NotReallyAValue'})
def get_auth_token_gss(account, gsstoken, appid, ip=None, session=None):
"""
Authenticate a Rucio account temporarily via a GSS token.
The token lifetime is 1 hour.
:param account: Account identifier as a string.
:param gsscred: GSS principal@REALM as a string.
:param appid: The application identifier as a string.
:param ip: IP address of the client as a string.
:param session: The database session in use.
:returns: Authentication token as a variable-length string.
"""
# Make sure the account exists
if not account_exists(account, session=session):
return None
# remove expired tokens
session.query(models.Token).filter(models.Token.expired_at < datetime.datetime.utcnow(), models.Token.account == account).delete()
# create new rucio-auth-token for account
tuid = generate_uuid() # NOQA
token = '%(account)s-%(gsstoken)s-%(appid)s-%(tuid)s' % locals()
new_token = models.Token(account=account, token=token, ip=ip)
new_token.save(session=session)
return token
def test_delete_dids(self):
""" DATA IDENTIFIERS (CORE): Delete dids """
tmp_scope = 'mock'
dsns = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'did_type': DIDType.DATASET} for i in xrange(5)]
for dsn in dsns:
add_did(scope=tmp_scope, name=dsn['name'], type='DATASET', account='root')
delete_dids(dids=dsns, account='root')
def get_auth_token_x509(account, dn, appid, ip=None, session=None):
"""
Authenticate a Rucio account temporarily via an x509 certificate.
The token lifetime is 1 hour.
:param account: Account identifier as a string.
:param dn: Client certificate distinguished name string, as extracted by Apache/mod_ssl.
:param appid: The application identifier as a string.
:param ip: IP address of the client as a string.
:param session: The database session in use.
:returns: Authentication token as a variable-length string.
"""
# Make sure the account exists
if not account_exists(account, session=session):
return None
# remove expired tokens
session.query(models.Token).filter(models.Token.expired_at < datetime.datetime.utcnow(), models.Token.account == account).delete()
# create new rucio-auth-token for account
tuid = generate_uuid() # NOQA
token = '%(account)s-%(dn)s-%(appid)s-%(tuid)s' % locals()
new_token = models.Token(account=account, token=token, ip=ip)
new_token.save(session=session)
return token
def setupClass(self):
self.c = ConfigClient()
self.test_section_1 = str(generate_uuid())
self.test_section_2 = str(generate_uuid())
self.test_option_s = 'string'
self.test_option_b = 'bool'
self.test_option_i = 'int'
self.test_option_f = 'float'
self.test_option_sv = 'iddqd'
self.test_option_bv = 'True'
self.test_option_iv = '543210'
self.test_option_fv = '3.1415'
self.c.set_config_option(self.test_section_1, self.test_option_s, self.test_option_sv)
self.c.set_config_option(self.test_section_1, self.test_option_b, self.test_option_bv)
self.c.set_config_option(self.test_section_2, self.test_option_i, self.test_option_iv)
self.c.set_config_option(self.test_section_2, self.test_option_f, self.test_option_fv)
def requeue_and_archive(request_id, session=None):
"""
Requeue and archive a failed request.
TODO: Multiple requeue.
:param request_id: Original request ID as a string.
:param session: Database session to use.
"""
record_counter('core.request.requeue_request')
new_req = get_request(request_id, session=session)
if new_req:
archive_request(request_id, session=session)
new_req['request_id'] = generate_uuid()
new_req['previous_attempt_id'] = request_id
if new_req['retry_count'] is None:
new_req['retry_count'] = 1
else:
new_req['retry_count'] += 1
# hardcoded for now - only requeue a couple of times
if new_req['retry_count'] < 4:
queue_requests([new_req], session=session)
return new_req
def test_touch_dids(self):
""" DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp"""
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root')
add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root')
now = datetime.utcnow()
now -= timedelta(microseconds=now.microsecond)
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET, 'accessed_at': now}])
assert_equal(now, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
def test_get_did(self):
""" DATA IDENTIFIERS (CLIENT): add a new data identifier and try to retrieve it back"""
rse = 'MOCK'
scope = 'mock'
file = generate_uuid()
dsn = generate_uuid()
self.replica_client.add_replica(rse, scope, file, 1L, '0cc737eb')
did = self.did_client.get_did(scope, file)
assert_equal(did['scope'], scope)
assert_equal(did['name'], file)
self.did_client.add_dataset(scope=scope, name=dsn, lifetime=10000000)
did2 = self.did_client.get_did(scope, dsn)
assert_equal(type(did2['expired_at']), datetime)
def test_add_datasets(self):
""" DATA IDENTIFIERS (CLIENT): Bulk add datasets """
tmp_scope = 'mock'
dsns = list()
for i in xrange(500):
tmp_dsn = {'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'meta': {'project': 'data13_hip'}}
dsns.append(tmp_dsn)
self.did_client.add_datasets(dsns)
def test_open(self):
""" DATA IDENTIFIERS (CLIENT): test to re-open data identifiers for priv account"""
tmp_rse = 'MOCK'
tmp_scope = 'mock'
# Add dataset
tmp_dataset = 'dsn_%s' % generate_uuid()
# Add file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(rse=tmp_rse, scope=tmp_scope, name=tmp_file, bytes=1L, adler32='0cc737eb')
# Add dataset
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dataset)
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1L, 'adler32': '0cc737eb'}, ]
def test_replica_http_redirection(self):
""" REPLICA (redirection): http redirection to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" -X GET %s/redirect/%s/%s''' % (self.cacert, self.token, self.host, tmp_scope, tmp_name)
exitcode, out, err = execute(cmd)
assert_in('404 Not Found', out)
# add replicas
self.replica_client.add_replicas(rse='MOCK', files=[{'scope': tmp_scope, 'name': tmp_name, 'bytes': 1L, 'adler32': '0cc737eb'}])
def test_detach_did(self):
""" DATA IDENTIFIERS (CLIENT): Detach dids from a did"""
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(2)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1L, '0cc737eb')
for i in range(4):
self.did_client.add_dataset(scope, dst[i], statuses=None, meta=None, rules=None)
for i in range(2):
self.did_client.add_container(scope, cnt[i], statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1L, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1L, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': dst[2]}, {'scope': scope, 'name': dst[3]}])
with assert_raises(UnsupportedOperation):
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': cnt[1]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
self.did_client.detach_dids(scope, cnt[0], [{'scope': scope, 'name': dst[1]}])
self.did_client.detach_dids(scope, dst[3], [{'scope': scope, 'name': file[6]}, {'scope': scope, 'name': file[7]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
if r['name'] == dst[1]:
assert_equal(r['level'], 0)
if r['type'] is 'file':
if (r['name'] in file[6:9]):
assert_equal(r['level'], 0)
else:
assert_not_equal(r['level'], 0)
with assert_raises(UnsupportedOperation):
self.did_client.detach_dids(scope=scope, name=cnt[0], dids=[{'scope': scope, 'name': cnt[0]}])
def test_get_did_from_pfns_deterministic(self):
""" REPLICA (CLIENT): Get list of DIDs associated to PFNs for deterministic sites"""
tmp_scope = 'mock'
rse = 'MOCK3'
nbfiles = 3
pfns = []
input = {}
rse_info = rsemgr.get_rse_info(rse)
assert_equal(rse_info['deterministic'], True)
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in xrange(nbfiles)]
def test_get_did_from_pfns_nondeterministic(self):
""" REPLICA (CLIENT): Get list of DIDs associated to PFNs for non-deterministic sites"""
rse = 'MOCK2'
tmp_scope = 'mock'
nbfiles = 3
pfns = []
input = {}
rse_info = rsemgr.get_rse_info(rse)
assert_equal(rse_info['deterministic'], False)
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb',
'pfn': 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), 'meta': {'events': 10}} for i in xrange(nbfiles)]
def test_update_replicas_paths(self):
""" REPLICA (CORE): Force update the replica path """
tmp_scope = 'mock'
nbfiles = 5
rse_info = rsemgr.get_rse_info('MOCK')
files = [{'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'pfn': 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests//does/not/really/matter/where',
'bytes': 1L,
'adler32': '0cc737eb',
'meta': {'events': 10},
'rse_id': rse_info['id'],
'path': '/does/not/really/matter/where'} for i in xrange(nbfiles)]
def test_list_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
for i in xrange(0, 5):
did.add_did(scope=tmp_scope, name='%s-%i' % (tmp_dsn, i), type='DATASET', issuer='root')
for i in did.list_new_dids('DATASET'):
assert_not_equal(i, {})
assert_equal(str(i['did_type']), 'DATASET')
break
for i in did.list_new_dids():
assert_not_equal(i, {})
break
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in xrange(3)]
self.tmp_scopes = [scope_name_generator() for i in xrange(3)]
self.tmp_rses = [rse_name_generator() for i in xrange(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in xrange(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in xrange(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in xrange(3)]
# add dummy data to the catalogue
for i in xrange(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1L, '0cc737eb')
# put files in datasets
for i in xrange(3):
for j in xrange(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1L, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
def test_update_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers and update the flag new """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
dids = []
for i in xrange(0, 5):
d = {'scope': tmp_scope, 'name': '%s-%i' % (tmp_dsn, i), 'did_type': DIDType.DATASET}
did.add_did(scope=tmp_scope, name='%s-%i' % (tmp_dsn, i), type='DATASET', issuer='root')
dids.append(d)
st = did.set_new_dids(dids, None)
assert_true(st)
with assert_raises(DataIdentifierNotFound):
did.set_new_dids([{'scope': 'dummyscope', 'name': 'dummyname', 'did_type': DIDType.DATASET}], None)
def test_add_dataset(self):
""" DATA IDENTIFIERS (CLIENT): Add dataset """
tmp_scope = 'mock'
tmp_dsn = 'dsn_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, meta={'project': 'data13_hip'})
did = self.did_client.get_did(tmp_scope, tmp_dsn)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_dsn)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_set_rse_limits(self):
""" RSE (CLIENTS): Test the update of RSE limits."""
nb_files = 30
file_size = 2147483648L # 2G
for file in xrange(nb_files):
replica_core.add_replica(rse='MOCK', scope='data13_hip', name='lfn' + generate_uuid(), bytes=file_size, account='root', adler32=None, md5=None)
rse_core.set_rse_usage(rse='MOCK', source='srm', used=nb_files*file_size, free=800L)
rse_core.set_rse_limits(rse='MOCK', name='MinFreeSpace', value=10737418240L)
rse_core.set_rse_limits(rse='MOCK', name='MaxBeingDeletedFiles', value=10)
rses = [rse_core.get_rse('MOCK'), ]
reaper(once=True, rses=rses)
reaper(once=True, rses=rses)
def test_exists(self):
""" DATA IDENTIFIERS (CLIENT): Check if data identifier exists """
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
tmp_rse = 'MOCK'
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L, '0cc737eb')
did = self.did_client.get_did(tmp_scope, tmp_file)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_file)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def generate_rse(endpoint, token):
rse_name = 'RSE%s' % generate_uuid().upper()
scheme = 'https'
impl = 'rucio.rse.protocols.webdav.Default'
if not endpoint.startswith('https://'):
scheme = 'srm'
impl = 'rucio.rse.protocols.srm.Default'
tmp_proto = {
'impl': impl,
'scheme': scheme,
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
}
rse_id = rse.add_rse(rse_name)
tmp_proto['hostname'] = endpoint.split(':')[1][2:]
tmp_proto['port'] = endpoint.split(':')[2].split('/')[0]
tmp_proto['prefix'] = '/'.join([''] +
endpoint.split(':')[2].split('/')[1:])
if scheme == 'srm':
tmp_proto['extended_attributes'] = {
'space_token': token,
'web_service_path': '/srm/managerv2?SFN='
}
rse.add_protocol(rse_id=rse_id, parameter=tmp_proto)
rse.add_rse_attribute(rse_id=rse_id,
key='fts',
value='https://fts3-pilot.cern.ch:8446')
account_limit.set_account_limit(
account=InternalAccount('root'),
rse_id=rsemanager.get_rse_info(rse_name)['id'],
bytes=-1)
return rsemanager.get_rse_info(rse_name)
def test_archive_of_deleted_dids(vo, did_factory, root_account, core_config_mock, caches_mock, file_config_mock):
""" REAPER (DAEMON): Test that the options to keep the did and content history work."""
[reaper_cache_region, _config_cache_region, _replica_cache_region] = caches_mock
scope = InternalScope('data13_hip', vo=vo)
account = root_account
nb_files = 10
file_size = 200 # 2G
rse_name, rse_id, dids = __add_test_rse_and_replicas(vo=vo, scope=scope, rse_name=rse_name_generator(),
names=['lfn' + generate_uuid() for _ in range(nb_files)], file_size=file_size, epoch_tombstone=True)
dataset = did_factory.make_dataset()
did_core.attach_dids(dids=dids, account=account, **dataset)
rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size)
assert len(list(replica_core.list_replicas(dids=dids, rse_expression=rse_name))) == nb_files
# Check first if the reaper does not delete anything if no space is needed
reaper_cache_region.invalidate()
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=323000000000)
reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None, greedy=True)
assert len(list(replica_core.list_replicas(dids=dids, rse_expression=rse_name))) == 0
file_clause = []
for did in dids:
file_clause.append(and_(models.DeletedDataIdentifier.scope == did['scope'], models.DeletedDataIdentifier.name == did['name']))
session = get_session()
query = session.query(models.DeletedDataIdentifier.scope,
models.DeletedDataIdentifier.name,
models.DeletedDataIdentifier.did_type).\
filter(or_(*file_clause))
deleted_dids = list()
for did in query.all():
print(did)
deleted_dids.append(did)
assert len(deleted_dids) == len(dids)
query = session.query(models.DataIdentifierAssociationHistory.child_scope,
models.DataIdentifierAssociationHistory.child_name,
models.DataIdentifierAssociationHistory.child_type).\
filter(and_(models.DataIdentifierAssociationHistory.scope == dataset['scope'], models.DataIdentifierAssociationHistory.name == dataset['name']))
deleted_dids = list()
for did in query.all():
print(did)
deleted_dids.append(did)
assert len(deleted_dids) == len(dids)
def test_get_meta(self):
""" DATA IDENTIFIERS (CLIENT): add a new meta data for an identifier and try to retrieve it back"""
rse = 'MOCK'
scope = 'mock'
file = generate_uuid()
keys = ['project', 'run_number']
values = ['data13_hip', 12345678]
self.replica_client.add_replica(rse, scope, file, 1, '0cc737eb')
for i in range(2):
self.did_client.set_metadata(scope, file, keys[i], values[i])
meta = self.did_client.get_metadata(scope, file)
for i in range(2):
assert_equal(meta[keys[i]], values[i])
def test_add_dataset(self):
""" DATA IDENTIFIERS (CLIENT): Add dataset """
tmp_scope = 'mock'
tmp_dsn = 'dsn_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope,
name=tmp_dsn,
meta={'project': 'data13_hip'})
did = self.did_client.get_did(tmp_scope, tmp_dsn)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_dsn)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_exists(self):
""" DATA IDENTIFIERS (CLIENT): Check if data identifier exists """
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
tmp_rse = 'MOCK'
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L,
'0cc737eb')
did = self.did_client.get_did(tmp_scope, tmp_file)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_file)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_delete_replicas(self):
""" REPLICA (CLIENT): Add and delete file replicas """
tmp_scope = 'mock'
nbfiles = 5
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK', files=files)
with assert_raises(AccessDenied):
self.replica_client.delete_replicas(rse='MOCK', files=files)
def test_replica_meta_redirection(self):
""" REDIRECT: metalink to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" -X GET %s/redirect/%s/%s' '' % (
self.cacert, self.token, self.host, tmp_scope, tmp_name)
_, out, _ = execute(cmd)
assert_in('404 Not Found', out)
# add replicas
self.replica_client.add_replicas(rse='MOCK',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1L,
'adler32': '0cc737eb'
}])
def test_update_lock_counter(self):
""" RSE (CORE): Test the update of a replica lock counter """
rse = 'MOCK'
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
add_replica(rse=rse, scope=tmp_scope, name=tmp_file, bytes=1, adler32='0cc737eb', account='jdoe')
values = (1, 1, 1, -1, -1, -1, 1, 1, -1)
tombstones = (True, True, True, True, True, False, True, True, True)
lock_counters = (1, 2, 3, 2, 1, 0, 1, 2, 1)
for value, tombstone, lock_counter in zip(values, tombstones, lock_counters):
status = update_replica_lock_counter(rse=rse, scope=tmp_scope, name=tmp_file, value=value)
assert_equal(status, True)
replica = get_replica(rse=rse, scope=tmp_scope, name=tmp_file)
assert_equal(replica['tombstone'] is None, tombstone)
assert_equal(lock_counter, replica['lock_cnt'])
def test_abacus_collection_replica(self):
""" ABACUS (COLLECTION REPLICA): Test update of collection replica. """
self.files = [{'did_scope': self.scope, 'did_name': 'file_' + generate_uuid(), 'path': file_generator(size=self.file_sizes), 'rse': self.rse, 'lifetime': -1} for i in range(0, 2)]
self.did_client.add_did(self.scope, self.dataset, DIDType.DATASET, lifetime=-1)
self.upload_client.upload(self.files)
self.did_client.attach_dids(scope=self.scope, name=self.dataset, dids=[{'name': file['did_name'], 'scope': file['did_scope']} for file in self.files])
self.rule_client.add_replication_rule([{'scope': self.scope, 'name': self.dataset}], 1, self.rse, lifetime=-1)
[os.remove(file['path']) for file in self.files]
# Check dataset replica after rule creation - initial data
dataset_replica = [replica for replica in self.replica_client.list_dataset_replicas(self.scope, self.dataset)][0]
assert_equal(dataset_replica['bytes'], 0)
assert_equal(dataset_replica['length'], 0)
assert_equal(dataset_replica['available_bytes'], 0)
assert_equal(dataset_replica['available_length'], 0)
assert_equal(str(dataset_replica['state']), 'UNAVAILABLE')
# Run Abacus
collection_replica.run(once=True)
# Check dataset replica after abacus - abacus should update the collection_replica table from updated_col_rep
dataset_replica = [replica for replica in self.replica_client.list_dataset_replicas(self.scope, self.dataset)][0]
assert_equal(dataset_replica['bytes'], len(self.files) * self.file_sizes)
assert_equal(dataset_replica['length'], len(self.files))
assert_equal(dataset_replica['available_bytes'], len(self.files) * self.file_sizes)
assert_equal(dataset_replica['available_length'], len(self.files))
assert_equal(str(dataset_replica['state']), 'AVAILABLE')
# Delete one file -> collection replica should be unavailable
cleaner.run(once=True)
delete_replicas(rse_id=self.rse_id, files=[{'name': self.files[0]['did_name'], 'scope': InternalScope(self.files[0]['did_scope'])}])
self.rule_client.add_replication_rule([{'scope': self.scope, 'name': self.dataset}], 1, self.rse, lifetime=-1)
collection_replica.run(once=True)
dataset_replica = [replica for replica in self.replica_client.list_dataset_replicas(self.scope, self.dataset)][0]
assert_equal(dataset_replica['length'], len(self.files))
assert_equal(dataset_replica['bytes'], len(self.files) * self.file_sizes)
assert_equal(dataset_replica['available_length'], len(self.files) - 1)
assert_equal(dataset_replica['available_bytes'], (len(self.files) - 1) * self.file_sizes)
assert_equal(str(dataset_replica['state']), 'UNAVAILABLE')
# Delete all files -> collection replica should be deleted
cleaner.run(once=True)
reaper.run(once=True, rses=[self.rse], greedy=True)
self.rule_client.add_replication_rule([{'scope': self.scope, 'name': self.dataset}], 1, self.rse, lifetime=-1)
collection_replica.run(once=True)
dataset_replica = [replica for replica in self.replica_client.list_dataset_replicas(self.scope, self.dataset)]
assert_equal(len(dataset_replica), 0)
def test_abacus_account(self):
""" ABACUS (ACCOUNT): Test update of account usage """
self.session.query(models.UpdatedAccountCounter).delete() # pylint: disable=no-member
self.session.query(models.AccountUsage).delete() # pylint: disable=no-member
self.session.commit() # pylint: disable=no-member
# Upload files -> account usage should increase
self.files = [{
'did_scope': self.scope.external,
'did_name': 'file_' + generate_uuid(),
'path': file_generator(size=self.file_sizes),
'rse': self.rse,
'lifetime': -1
} for i in range(0, 2)]
self.upload_client.upload(self.files)
[os.remove(file['path']) for file in self.files]
account.run(once=True)
account_usage = get_local_account_usage(account=self.account,
rse_id=self.rse_id)[0]
assert account_usage['bytes'] == len(self.files) * self.file_sizes
assert account_usage['files'] == len(self.files)
# Update and check the account history with the core method
update_account_counter_history(account=self.account,
rse_id=self.rse_id)
usage_history = get_usage_history(rse_id=self.rse_id,
account=self.account)
assert usage_history[-1]['bytes'] == len(self.files) * self.file_sizes
assert usage_history[-1]['files'] == len(self.files)
# Check the account history with the client
usage_history = self.account_client.get_account_usage_history(
rse=self.rse, account=self.account.external)
assert usage_history[-1]['bytes'] == len(self.files) * self.file_sizes
assert usage_history[-1]['files'] == len(self.files)
# Delete rules -> account usage should decrease
cleaner.run(once=True)
account.run(once=True)
# set account limit because return value of get_local_account_usage differs if a limit is set or not
set_local_account_limit(account=self.account,
rse_id=self.rse_id,
bytes=10)
account_usages = get_local_account_usage(account=self.account,
rse_id=self.rse_id)[0]
assert account_usages['bytes'] == 0
assert account_usages['files'] == 0
def test_download_to_two_paths(rse_factory, did_factory, download_client):
scope = str(did_factory.default_scope)
rse, _ = rse_factory.make_posix_rse()
base_name = generate_uuid()
item000 = did_factory.upload_test_file(rse,
name=base_name + '.000',
return_full_item=True)
item001 = did_factory.upload_test_file(rse,
name=base_name + '.001',
return_full_item=True)
item100 = did_factory.upload_test_file(rse,
name=base_name + '.100',
return_full_item=True)
did000_str = '%s:%s' % (item000['did_scope'], item000['did_name'])
did001_str = '%s:%s' % (item001['did_scope'], item001['did_name'])
did100_str = '%s:%s' % (item100['did_scope'], item100['did_name'])
with TemporaryDirectory() as tmp_dir1, TemporaryDirectory() as tmp_dir2:
# Download two overlapping wildcard dids to two separate paths.
# 000 will be in both paths. Other two files only in one of the two paths.
result = download_client.download_dids([{
'did':
'%s:%s.*0' % (scope, base_name),
'base_dir':
tmp_dir1
}, {
'did':
'%s:%s.0*' % (scope, base_name),
'base_dir':
tmp_dir2
}])
paths000 = next(filter(lambda r: r['did'] == did000_str,
result))['dest_file_paths']
paths001 = next(filter(lambda r: r['did'] == did001_str,
result))['dest_file_paths']
paths100 = next(filter(lambda r: r['did'] == did100_str,
result))['dest_file_paths']
assert len(paths000) == 2
assert any(p.startswith(tmp_dir1) for p in paths000)
assert any(p.startswith(tmp_dir2) for p in paths000)
assert len(paths001) == 1
assert paths001[0].startswith(tmp_dir2)
assert len(paths100) == 1
assert paths100[0].startswith(tmp_dir1)
def test_abacus_rse(self):
""" ABACUS (RSE): Test update of RSE usage. """
# Get RSE usage of all sources
self.session.query(models.UpdatedRSECounter).delete() # pylint: disable=no-member
self.session.query(models.RSEUsage).delete() # pylint: disable=no-member
self.session.commit() # pylint: disable=no-member
# Upload files -> RSE usage should increase
self.files = [{
'did_scope': self.scope,
'did_name': 'file_' + generate_uuid(),
'path': file_generator(size=self.file_sizes),
'rse': self.rse,
'lifetime': -1
} for i in range(0, 2)]
self.upload_client.upload(self.files)
[os.remove(file['path']) for file in self.files]
rse.run(once=True)
rse_usage = get_rse_usage(rse_id=self.rse_id)[0]
assert rse_usage['used'] == len(self.files) * self.file_sizes
rse_usage_from_rucio = get_rse_usage(rse_id=self.rse_id,
source='rucio')[0]
assert rse_usage_from_rucio['used'] == len(
self.files) * self.file_sizes
rse_usage_from_unavailable = get_rse_usage(rse_id=self.rse_id,
source='unavailable')
assert len(rse_usage_from_unavailable) == 0
# Delete files -> rse usage should decrease
from rucio.daemons.reaper.reaper2 import REGION
REGION.invalidate()
cleaner.run(once=True)
if self.vo:
reaper2.run(once=True,
include_rses='vo=%s&(%s)' % (self.vo['vo'], self.rse),
greedy=True)
else:
reaper2.run(once=True, include_rses=self.rse, greedy=True)
rse.run(once=True)
rse_usage = get_rse_usage(rse_id=self.rse_id)[0]
assert rse_usage['used'] == 0
rse_usage_from_rucio = get_rse_usage(rse_id=self.rse_id,
source='rucio')[0]
assert rse_usage_from_rucio['used'] == 0
rse_usage_from_unavailable = get_rse_usage(rse_id=self.rse_id,
source='unavailable')
assert len(rse_usage_from_unavailable) == 0
def test_abacus_collection_replica_cleanup(self):
""" ABACUS (COLLECTION REPLICA): Test if the cleanup procedure works correctly. """
collection_replica.run(once=True)
db_session = session.get_session()
rse1 = rse_name_generator()
rse_id1 = add_rse(rse1, **self.vo)
rse2 = rse_name_generator()
rse_id2 = add_rse(rse2, **self.vo)
scope = InternalScope('mock', **self.vo)
dataset = 'dataset_%s' % generate_uuid()
jdoe = InternalAccount('jdoe', **self.vo)
add_did(scope, dataset, DIDType.DATASET, jdoe)
models.CollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id1,
state=ReplicaState.AVAILABLE,
bytes=1).save(session=db_session, flush=False)
models.CollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id2,
state=ReplicaState.AVAILABLE,
bytes=1).save(session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id1).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id1).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id2).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id2).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope, name=dataset,
rse_id=None).save(session=db_session,
flush=False)
db_session.commit()
assert len(get_cleaned_updated_collection_replicas(1, 1)) == 3
def get_file_guid(self, file):
guid = file.get('guid')
if not guid and 'pool.root' in file['basename'].lower() and not file.get('no_register'):
status, output, err = execute('pool_extractFileIdentifier %s' % file['path'])
if status != 0:
msg = 'Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.\n'
msg += 'Setup your ATHENA environment and try again.'
raise RucioException(msg)
try:
guid = output.splitlines()[-1].split()[0].replace('-', '').lower()
except Exception:
raise RucioException('Error extracting GUID from ouput of pool_extractFileIdentifier')
elif guid:
guid = guid.replace('-', '')
else:
guid = generate_uuid()
return guid
def test_list_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
for i in xrange(0, 5):
did.add_did(scope=tmp_scope,
name='%s-%i' % (tmp_dsn, i),
type='DATASET',
issuer='root')
for i in did.list_new_dids('DATASET'):
assert_not_equal(i, {})
assert_equal(str(i['did_type']), 'DATASET')
break
for i in did.list_new_dids():
assert_not_equal(i, {})
break
def test_upload_file_with_supported_protocol(rse_factory, upload_client, mock_scope, file_factory):
""" Upload (CLIENT): Ensure the module associated to the first protocol supported by both the remote and local config is called """
rse_name, rse_id = rse_factory.make_rse()
# FIXME:
# The correct order to test should actually be ssh,xrootd,posix
# However the preferred_impl is not working correctly.
# Once preferred_impl is fixed, this should be changed back
add_protocol(rse_id, {'scheme': 'scp',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.posix.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1}}})
add_protocol(rse_id, {'scheme': 'root',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 2, 'write': 2, 'delete': 2},
'wan': {'read': 2, 'write': 2, 'delete': 2}}})
add_protocol(rse_id, {'scheme': 'file',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.ssh.Default',
'domains': {
'lan': {'read': 3, 'write': 3, 'delete': 3},
'wan': {'read': 3, 'write': 3, 'delete': 3}}})
path = file_factory.file_generator()
name = os.path.basename(path)
item = {
'path': path,
'rse': rse_name,
'did_scope': str(mock_scope),
'did_name': name,
'guid': generate_uuid()
}
status = upload_client.upload([item])
assert status == 0
def get_auth_token_user_pass(account, username, password, appid, ip=None, session=None):
"""
Authenticate a Rucio account temporarily via username and password.
The token lifetime is 1 hour.
:param account: Account identifier as a string.
:param username: Username as a string.
:param password: SHA1 hash of the password as a string.
:param appid: The application identifier as a string.
:param ip: IP address of the client a a string.
:param session: The database session in use.
:returns: A dict with token and expires_at entries.
"""
# Make sure the account exists
if not account_exists(account, session=session):
return None
result = session.query(models.Identity).filter_by(identity=username,
identity_type=IdentityType.USERPASS).first()
db_salt = result['salt']
db_password = result['password']
salted_password = db_salt + password.encode()
if db_password != hashlib.sha256(salted_password).hexdigest():
return None
# get account identifier
result = session.query(models.IdentityAccountAssociation).filter_by(identity=username,
identity_type=IdentityType.USERPASS,
account=account).first()
db_account = result['account']
# remove expired tokens
__delete_expired_tokens_account(account=account, session=session)
# create new rucio-auth-token for account
tuid = generate_uuid() # NOQA
token = '%(account)s-%(username)s-%(appid)s-%(tuid)s' % locals()
new_token = models.Token(account=db_account, identity=username, token=token, ip=ip)
new_token.save(session=session)
return token_dictionary(new_token)
def upload_test_file(self, rse_name, scope=None, name=None, path=None, return_full_item=False):
scope = self._sanitize_or_set_scope(scope)
if not path:
path = file_generator()
if not name:
name = os.path.basename(path)
item = {
'path': path,
'rse': rse_name,
'did_scope': str(scope),
'did_name': name,
'guid': generate_uuid(),
}
self.upload_client.upload([item])
did = {'scope': scope, 'name': name}
self.created_dids.append(did)
return item if return_full_item else did
def dataset(db_session, vo):
scope = InternalScope(scope='mock', vo=vo)
name = generate_uuid()
account = InternalAccount('root', vo=vo)
kwargs = {'scope': scope, 'name': name, 'type': DIDType.DATASET, 'account': account}
add_did(**kwargs, session=db_session)
db_session.commit()
yield kwargs
kwargs['did_type'] = kwargs['type']
del kwargs['type']
del kwargs['account']
kwargs['purge_replicas'] = True
delete_dids(dids=[kwargs], account=account)
db_session.commit()
def test_throttler_fifo_release_nothing(self):
""" THROTTLER (CLIENTS): throttler release nothing (fifo). """
# two waiting requests and one active requests but threshold is 1
# more than 80% of the transfer limit are already used -> release nothing
set('throttler', '%s,%s' % (self.user_activity, self.dest_rse), 1, session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.user_activity, state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
name1 = generate_uuid()
name2 = generate_uuid()
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name2,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request = get_request_by_did(self.scope, name1, self.dest_rse_id)
assert_equal(request['state'], constants.RequestState.WAITING)
request2 = get_request_by_did(self.scope, name2, self.dest_rse_id)
assert_equal(request2['state'], constants.RequestState.WAITING)
def test_account_counters_at_different_vos(self):
""" MULTI VO (CLIENT): Test that account counters from 2nd vo don't interfere """
session = db_session.get_session()
# add some RSEs to test create_counters_for_new_account
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst_rse1 = 'TST1_%s' % rse_str
new_rse1 = 'NEW1_%s' % rse_str
rse_client.add_rse(tst_rse1)
add_rse(new_rse1, 'root', **self.new_vo)
# add an account - should have counters created for RSEs on the same VO
usr_uuid = str(generate_uuid()).lower()[:16]
new_acc_str = 'shr-%s' % usr_uuid
new_acc = InternalAccount(new_acc_str, **self.new_vo)
add_account(new_acc_str, 'USER', '*****@*****.**', 'root', **self.new_vo)
query = session.query(models.AccountUsage.account, models.AccountUsage.rse_id).\
distinct(models.AccountUsage.account, models.AccountUsage.rse_id).\
filter_by(account=new_acc)
acc_counters = list(query.all())
assert_not_equal(0, len(acc_counters))
for counter in acc_counters:
rse_id = counter[1]
vo = get_rse_vo(rse_id)
assert_equal(vo, self.new_vo['vo'])
# add an RSE - should have counters created for accounts on the same VO
new_rse2 = 'NEW2_' + rse_str
new_rse2_id = add_rse(new_rse2, 'root', **self.new_vo)
query = session.query(models.AccountUsage.account, models.AccountUsage.rse_id).\
distinct(models.AccountUsage.account, models.AccountUsage.rse_id).\
filter_by(rse_id=new_rse2_id)
rse_counters = list(query.all())
assert_not_equal(0, len(rse_counters))
for counter in rse_counters:
account = counter[0]
assert_equal(account.vo, self.new_vo['vo'])
session.commit()
def test_get_did_from_pfns_deterministic(self):
""" REPLICA (CLIENT): Get list of DIDs associated to PFNs for deterministic sites"""
tmp_scope = 'mock'
rse = 'MOCK3'
nbfiles = 3
pfns = []
input = {}
rse_info = rsemgr.get_rse_info(rse)
assert_equal(rse_info['deterministic'], True)
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1L,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for i in xrange(nbfiles)]
def test_update_replicas_paths(self):
""" REPLICA (CORE): Force update the replica path """
tmp_scope = 'mock'
nbfiles = 5
rse_info = rsemgr.get_rse_info('MOCK')
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'pfn':
'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/does/not/really/matter/where',
'bytes': 1L,
'adler32': '0cc737eb',
'meta': {
'events': 10
},
'rse_id': rse_info['id'],
'path': '/does/not/really/matter/where'
} for i in xrange(nbfiles)]
def test_upload_file_with_impl(rse_factory, upload_client, mock_scope, file_factory):
""" Upload (CLIENT): Ensure the module associated to the impl value is called """
impl = 'xrootd'
rse_name, rse_id = rse_factory.make_rse()
add_protocol(rse_id, {'scheme': 'file',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.posix.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1}}})
add_protocol(rse_id, {'scheme': 'root',
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 2, 'write': 2, 'delete': 2},
'wan': {'read': 2, 'write': 2, 'delete': 2}}})
path = file_factory.file_generator()
name = os.path.basename(path)
item = {
'path': path,
'rse': rse_name,
'did_scope': str(mock_scope),
'did_name': name,
'guid': generate_uuid(),
'impl': impl
}
with TemporaryDirectory() as tmp_dir:
with patch('rucio.rse.protocols.%s.Default.put' % impl, side_effect=lambda pfn, dest, dir, **kw: shutil.copy(path, tmp_dir)) as mock_put, \
patch('rucio.rse.protocols.%s.Default.connect' % impl),\
patch('rucio.rse.protocols.%s.Default.exists' % impl, side_effect=lambda pfn, **kw: False),\
patch('rucio.rse.protocols.%s.Default.delete' % impl),\
patch('rucio.rse.protocols.%s.Default.rename' % impl),\
patch('rucio.rse.protocols.%s.Default.stat' % impl, side_effect=lambda pfn: {'filesize': os.stat(path)[os.path.stat.ST_SIZE], 'adler32': adler32(path)}),\
patch('rucio.rse.protocols.%s.Default.close' % impl):
mock_put.__name__ = "mock_put"
upload_client.upload([item])
mock_put.assert_called()
def test_reaper():
""" REAPER (DAEMON): Test the reaper daemon."""
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
vo = {}
rse_name = rse_name_generator()
rse_id = rse_core.add_rse(rse_name, **vo)
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/reaper',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
nb_files = 30
file_size = 2147483648 # 2G
file_names = []
for i in range(nb_files):
file_name = 'lfn' + generate_uuid()
file_names.append(file_name)
replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo),
name=file_name, bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **vo), adler32=None, md5=None)
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800)
rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
rses = [rse_core.get_rse(rse_id), ]
reaper(once=True, rses=rses)
reaper(once=True, rses=rses)
assert_equal(len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in file_names], rse_expression=rse_name))), nb_files - 10)
def download_aria2c(self, items, trace_custom_fields={}):
"""
Uses aria2c to download the items with given DIDs. This function can also download datasets and wildcarded DIDs.
It only can download files that are available via https/davs.
Aria2c needs to be installed and X509_USER_PROXY needs to be set!
:param items: List of dictionaries. Each dictionary describing an item to download. Keys:
did - DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
rse - Optional: rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
base_dir - Optional: base directory where the downloaded files will be stored. (Default: '.')
no_subdir - Optional: If true, files are written directly into base_dir and existing files are overwritten. (Default: False)
nrandom - Optional: if the DID addresses a dataset, nrandom files will be randomly choosen for download from the dataset
ignore_checksum - Optional: If true, skips the checksum validation between the downloaded file and the rucio catalouge. (Default: False)
:param trace_custom_fields: Custom key value pairs to send with the traces
:returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
:raises InputValidationError: if one of the input items is in the wrong format
:raises NoFilesDownloaded: if no files could be downloaded
:raises NotAllFilesDownloaded: if not all files could be downloaded
:raises RucioException: if something went wrong during the download (e.g. aria2c could not be started)
"""
trace_custom_fields['uuid'] = generate_uuid()
rpc_secret = '%x' % (random.getrandbits(64))
rpc_auth = 'token:' + rpc_secret
rpcproc, aria_rpc = self._start_aria2c_rpc(rpc_secret)
for item in items:
item['force_scheme'] = ['https', 'davs']
input_items = self._prepare_items_for_download(items)
try:
output_items = self._download_items_aria2c(input_items, aria_rpc, rpc_auth, trace_custom_fields)
except Exception as error:
self.logger.error('Unknown exception during aria2c download')
self.logger.debug(error)
finally:
try:
aria_rpc.aria2.forceShutdown(rpc_auth)
finally:
rpcproc.terminate()
return self._check_output(output_items)
def test_get_and_set_section_option(self):
""" CONFIG (CORE): Retreive configuration option only """
# get and set
section = str(generate_uuid())
option = str(generate_uuid())
expected_value = str(generate_uuid())
config_core.set(section=section, option=option, value=expected_value)
value = config_core.get(section, option, use_cache=False)
assert value == expected_value
# default value
section = str(generate_uuid())
config_core.set(section=section, option=str(generate_uuid()), value=str(generate_uuid()))
default_value = 'default'
value = config_core.get(section, 'new_option', default=default_value, use_cache=False)
assert value == default_value
# key with space character
section = str(generate_uuid() + ' ')
option = str(generate_uuid() + ' ')
expected_value = str(generate_uuid())
config_core.set(section=section, option=option, value=expected_value)
value = config_core.get(section, option, use_cache=False)
assert value == expected_value
def test_get_did_from_pfns_deterministic(self):
""" REPLICA (CLIENT): Get list of DIDs associated to PFNs for deterministic sites"""
tmp_scope = 'mock'
rse = 'MOCK3'
nbfiles = 3
pfns = []
input = {}
rse_info = rsemgr.get_rse_info(rse)
assert_equal(rse_info['deterministic'], True)
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)]
p = rsemgr.create_protocol(rse_info, 'read', scheme='srm')
for f in files:
pfn = p.lfns2pfns(lfns={'scope': f['scope'], 'name': f['name']}).values()[0]
pfns.append(pfn)
input[pfn] = {'scope': f['scope'], 'name': f['name']}
add_replicas(rse=rse, files=files, account='root', ignore_availability=True)
for result in self.replica_client.get_did_from_pfns(pfns, rse):
pfn = result.keys()[0]
assert_equal(input[pfn], result.values()[0])
def test_accounts_at_different_vos(self):
""" MULTI VO (CLIENT): Test that accounts from 2nd vo don't interfere """
account_client = AccountClient()
usr_uuid = str(generate_uuid()).lower()[:16]
tst = 'tst-%s' % usr_uuid
new = 'new-%s' % usr_uuid
shr = 'shr-%s' % usr_uuid
account_client.add_account(tst, 'USER', '*****@*****.**')
account_client.add_account(shr, 'USER', '*****@*****.**')
add_account(new, 'USER', '*****@*****.**', 'root', **self.new_vo)
add_account(shr, 'USER', '*****@*****.**', 'root', **self.new_vo)
account_list_tst = [a['account'] for a in account_client.list_accounts()]
account_list_new = [a['account'] for a in list_accounts(filter={}, **self.new_vo)]
assert_true(tst in account_list_tst)
assert_false(new in account_list_tst)
assert_true(shr in account_list_tst)
assert_false(tst in account_list_new)
assert_true(new in account_list_new)
assert_true(shr in account_list_new)
def test_scopes_at_different_vos(self):
""" MULTI VO (CLIENT): Test that scopes from 2nd vo don't interfere """
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
tst = 'tst_%s' % scope_uuid
new = 'new_%s' % scope_uuid
shr = 'shr_%s' % scope_uuid
scope_client.add_scope('root', tst)
scope_client.add_scope('root', shr)
add_scope(new, 'root', 'root', **self.new_vo)
add_scope(shr, 'root', 'root', **self.new_vo)
scope_list_tst = list(scope_client.list_scopes())
scope_list_new = list(list_scopes(filter={}, **self.new_vo))
assert_true(tst in scope_list_tst)
assert_false(new in scope_list_tst)
assert_true(shr in scope_list_tst)
assert_false(tst in scope_list_new)
assert_true(new in scope_list_new)
assert_true(shr in scope_list_new)