def setup(self):
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.base_client = BaseClient(account='root',
ca_cert=config_get('client', 'ca_cert'),
auth_type='x509')
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.fname = generate_uuid()
rses = ['MOCK', 'MOCK3', 'MOCK4']
dsn = generate_uuid()
self.files = [{
'scope': 'mock',
'name': self.fname,
'bytes': 1,
'adler32': '0cc737eb'
}]
self.did_client.add_dataset(scope='mock', name=dsn)
self.did_client.add_files_to_dataset('mock',
name=dsn,
files=self.files,
rse='MOCK')
for r in rses:
self.replica_client.add_replicas(r, self.files)
def setUp(self):
self.account = 'root'
self.scope = 'mock'
self.rse = 'MOCK5'
self.file_sizes = 2
self.dataset = 'dataset_%s' % generate_uuid()
self.rule_client = RuleClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.upload_client = UploadClient()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo = {}
self.rse_id = get_rse_id(rse=self.rse, **self.vo)
class TestReplicaMetalinkRedirection:
def __init__(self):
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.marker = '$> '
# get auth token
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def test_replica_meta_redirection(self):
""" REDIRECT: metalink to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" -X GET %s/redirect/%s/%s' '' % (
self.cacert, self.token, self.host, tmp_scope, tmp_name)
_, out, _ = execute(cmd)
assert_in('404 Not Found', out)
# add replicas
self.replica_client.add_replicas(rse='MOCK',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1L,
'adler32': '0cc737eb'
}])
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
class TestDIDClients(object):
def __init__(self):
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
def test_add_and_list_archive(self):
""" ARCHIVE (CLIENT): Add files to archive and list the content."""
scope, rse = 'mock', 'MOCK'
archive_file = 'file_' + generate_uuid() + '.zip'
files = []
for i in xrange(10):
files.append({
'scope': scope,
'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570L,
'adler32': '0cc737eb',
'type': 'FILE',
'meta': {
'guid': str(generate_uuid())
}
})
self.replica_client.add_replicas(rse=rse,
files=[{
'scope': scope,
'name': archive_file,
'bytes': 1L,
'adler32': '0cc737eb'
}])
def test_list_dataset_replicas_bulk(self):
""" REPLICA (CLIENT): List dataset replicas bulk."""
replica_client = ReplicaClient()
rule_client = RuleClient()
did_client = DIDClient()
scope = 'mock'
did1 = {'scope': scope, 'name': 'dataset_' + str(generate_uuid())}
did_client.add_dataset(**did1)
did2 = {'scope': scope, 'name': 'dataset_' + str(generate_uuid())}
did_client.add_dataset(**did2)
dids = [did1, did2]
rule_client.add_replication_rule(dids=dids,
account='root',
copies=1,
rse_expression='MOCK',
grouping='DATASET')
with pytest.raises(InvalidObject):
replica_client.list_dataset_replicas_bulk(
dids=[{
'type': "I'm Different"
}])
replicas = list(replica_client.list_dataset_replicas_bulk(dids=dids))
assert len(replicas) == 2
for did in dids:
def replica_contains_did(rep):
return all(map(lambda k: k in rep and did[k] == rep[k], did))
assert any(
map(replica_contains_did,
replicas)), "%s must be in returned replicas" % (did, )
def __init__(self,
dataset,
site,
rse=None,
scope=DEFAULT_SCOPE,
uuid=None,
check=True,
lifetime=None,
dry_run=False):
self.dataset = dataset
self.site = site
if rse is None:
rse = site
self.rse = rse
self.scope = scope
self.uuid = uuid
self.check = check
self.lifetime = lifetime
self.dry_run = dry_run
self.blocks = []
self.url = ''
self.getmetadata()
self.get_global_url()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.gfal = Gfal2Context()
def __init__(self):
#Take all data types categories
self.RAW_RECORDS_TPC_TYPES = helper.get_hostconfig(
)['raw_records_tpc_types']
self.RAW_RECORDS_MV_TYPES = helper.get_hostconfig(
)['raw_records_mv_types']
self.RAW_RECORDS_NV_TYPES = helper.get_hostconfig(
)['raw_records_nv_types']
self.LIGHT_RAW_RECORDS_TPC_TYPES = helper.get_hostconfig(
)['light_raw_records_tpc_types']
self.LIGHT_RAW_RECORDS_MV_TYPES = helper.get_hostconfig(
)['light_raw_records_mv_types']
self.LIGHT_RAW_RECORDS_NV_TYPES = helper.get_hostconfig(
)['light_raw_records_nv_types']
self.HIGH_LEVEL_TYPES = helper.get_hostconfig()['high_level_types']
self.RECORDS_TYPES = helper.get_hostconfig()['records_types']
#Choose which data type you want to treat
self.DTYPES = self.RAW_RECORDS_TPC_TYPES + self.RAW_RECORDS_MV_TYPES + self.RAW_RECORDS_NV_TYPES + self.LIGHT_RAW_RECORDS_TPC_TYPES + self.LIGHT_RAW_RECORDS_MV_TYPES + self.LIGHT_RAW_RECORDS_NV_TYPES + self.HIGH_LEVEL_TYPES + self.RECORDS_TYPES
#Take the list of all XENON RSEs
self.RSES = helper.get_hostconfig()['rses']
#Take the RSE that is used to perform the upload
self.UPLOAD_TO = helper.get_hostconfig()['upload_to']
#Take the directory where datamanager has to upload data
self.DATADIR = helper.get_hostconfig()['path_data_to_upload']
# Get the sequence of rules to be created according to the data type
self.RAW_RECORDS_TPC_RSES = helper.get_hostconfig(
)["raw_records_tpc_rses"]
self.RAW_RECORDS_MV_RSES = helper.get_hostconfig(
)["raw_records_mv_rses"]
self.RAW_RECORDS_NV_RSES = helper.get_hostconfig(
)["raw_records_nv_rses"]
self.LIGHT_RAW_RECORDS_TPC_RSES = helper.get_hostconfig(
)["light_raw_records_tpc_rses"]
self.LIGHT_RAW_RECORDS_MV_RSES = helper.get_hostconfig(
)["light_raw_records_mv_rses"]
self.LIGHT_RAW_RECORDS_NV_RSES = helper.get_hostconfig(
)["light_raw_records_nv_rses"]
self.HIGH_LEVEL_RSES = helper.get_hostconfig()["high_level_rses"]
self.RECORDS_RSES = helper.get_hostconfig()["records_rses"]
#Init the runDB
self.db = ConnectMongoDB()
#Init Rucio for later uploads and handling:
self.rc = RucioSummoner()
self.didclient = DIDClient()
self.replicaclient = ReplicaClient()
#Rucio Rule assignment priority
self.priority = 3
#Parameters to write warnings
self.minimum_number_acceptable_rses = 2
self.minimum_deltadays_allowed = 3
def __init__(self):
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def __init__(self):
self.dc = DIDClient()
self.rc = ReplicaClient()
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
class TestReplicaMetalinkRedirection:
def __init__(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo_header = '-H "X-Rucio-VO: %s"' % config_get(
'client', 'vo', raise_exception=False, default='tst')
else:
self.vo_header = ''
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def test_replica_meta_redirection(self):
""" REDIRECT: metalink to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s' % (
self.cacert, self.token, self.vo_header, self.host, tmp_scope,
tmp_name)
_, out, _ = execute(cmd)
assert_in('404 Not Found', out)
self.replica_client.add_replicas(rse='MOCK',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'
}])
self.replica_client.add_replicas(rse='MOCK3',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'
}])
_, out, _ = execute(cmd)
assert_in('303 See Other', out)
assert_in('Link: </redirect/%s/%s/metalink' % (tmp_scope, tmp_name),
out)
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s/metalink' % (
self.cacert, self.token, self.vo_header, self.host, tmp_scope,
tmp_name)
_, out, _ = execute(cmd)
assert_in('200 OK', out)
assert_in('<?xml', out)
assert_in('<metalink', out)
assert_in('<url location="MOCK"', out)
assert_in('<url location="MOCK3"', out)
def setUp(self):
self.account = 'root'
self.scope = 'mock'
self.rule_client = RuleClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.upload_client = UploadClient()
self.file_sizes = 2
self.dataset = 'dataset_%s' % generate_uuid()
self.rse = 'MOCK5'
self.rse_id = get_rse_id(rse=self.rse)
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def getReplicaInfoForBlocks(self,
scope='cms',
dataset=None,
block=None,
node=None): # Mirroring PhEDEx service
"""
_blockreplicas_
Get replicas for given blocks
dataset dataset name, can be multiple (*)
block block name, can be multiple (*)
node node name, can be multiple (*)
se storage element name, can be multiple (*)
update_since unix timestamp, only return replicas updated since this
time
create_since unix timestamp, only return replicas created since this
time
complete y or n, whether or not to require complete or incomplete
blocks. Default is to return either
subscribed y or n, filter for subscription. default is to return either.
custodial y or n. filter for custodial responsibility. default is
to return either.
group group name. default is to return replicas for any group.
"""
block_names = []
result = {'block': []}
rc = ReplicaClient(account=self.account, auth_type=self.auth_type)
if isinstance(block, (list, set)):
block_names = block
elif block:
block_names = [block]
if isinstance(dataset, (list, set)):
for dataset_name in dataset:
block_names.extend(
self.cmsBlocksInContainer(dataset_name, scope=scope))
elif dataset:
block_names.extend(self.cmsBlocksInContainer(dataset, scope=scope))
for block_name in block_names:
dids = [{'scope': scope, 'name': block_name}]
response = rc.list_replicas(dids=dids)
nodes = set()
for item in response:
for node, state in item['states'].items():
if state.upper() == 'AVAILABLE':
nodes.add(node)
result['block'].append({block_name: list(nodes)})
return result
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo_header = '-H "X-Rucio-VO: %s"' % config_get('client', 'vo', raise_exception=False, default='tst')
else:
self.vo_header = ''
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def test_list_dataset_replicas(self):
""" REPLICA (CLIENT): List dataset replicas."""
replica_client = ReplicaClient()
rule_client = RuleClient()
did_client = DIDClient()
scope = 'mock'
dataset = 'dataset_' + str(generate_uuid())
did_client.add_dataset(scope=scope, name=dataset)
rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}],
account='root', copies=1, rse_expression='MOCK',
grouping='DATASET')
replicas = [r for r in replica_client.list_dataset_replicas(scope=scope, name=dataset)]
assert len(replicas) == 1
def setup(self):
try:
remove('/tmp/.rucio_root/auth_token_root')
except OSError as e:
if e.args[0] != 2:
raise e
self.marker = '$> '
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.user = '******'
self.def_rse = 'MOCK4'
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.account_client = AccountLimitClient()
self.account_client.set_account_limit('root', self.def_rse, -1)
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
class TestReplicaHeaderRedirection(unittest.TestCase):
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo_header = '-H "X-Rucio-VO: %s"' % config_get(
'client', 'vo', raise_exception=False, default='tst')
else:
self.vo_header = ''
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def test_replica_header_redirection(self):
""" REDIRECT: header to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s' '' % (
self.cacert, self.token, self.vo_header, self.host, tmp_scope,
tmp_name)
_, out, _ = execute(cmd)
assert '404 Not Found'.lower() in out.lower()
self.replica_client.add_replicas(rse='MOCK',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'
}])
self.replica_client.add_replicas(rse='MOCK3',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'
}])
_, out, _ = execute(cmd)
assert '303 See Other'.lower() in out.lower()
assert 'Location: https://mock' in out
def test_get_vo_from_config(self):
""" MULTI VO (CLIENT): Get vo from config file when starting clients """
# Start clients with vo explicitly set to None
replica_client = ReplicaClient(vo=None)
client = Client(vo=None)
upload_client = UploadClient(_client=client)
# Check the vo has been got from the config file
assert_equal(replica_client.vo, self.vo['vo'])
assert_equal(upload_client.client.vo, self.vo['vo'])
def setup(self):
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.marker = '$> '
# get auth token
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def __init__(self,
run,
originrse=DEFAULT_ORIGIN_RSE,
destrse=None,
scope=DEFAULT_SCOPE,
check=True,
lifetime=None,
dry_run=False,
container=None):
"""
:param dataset: Name of the PhEDEx dataset to synchronize with Rucio.
:param pnn: PhEDEx node name to filter on for replica information.
"""
self.run = run
self.originrse = originrse
self.destrse = destrse
self.scope = scope
self.check = check
self.lifetime = lifetime
self.dry_run = dry_run
self.container = container
self.rucio_datasets = {}
self.run_files = {}
self.existent_replica_files = {}
self.url = ''
self.gfal = Gfal2Context()
self.run_Number = None
self.get_run_Number()
self.files_storage = {}
self.get_global_url()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.rulesClient = RuleClient()
# Right now obtaining the Metadata from the storage at WIPAC
# Hopefully in the future from JADE # TODO
self.get_run_Files()
self.get_rucio_metadata()
self.update_run_Files()
self.get_files_metadata()
def setup(self):
try:
remove('/tmp/.rucio_root/auth_token_root')
except OSError as e:
if e.args[0] != 2:
raise e
self.marker = '$> '
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.user = '******'
self.def_rse = 'MOCK4'
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.account_client = AccountLimitClient()
self.account_client.set_account_limit('root', self.def_rse, -1)
add_rse_attribute(self.def_rse, 'istape', 'False')
self.upload_success_str = 'Successfully uploaded file %s'
def rucio_register(self, filenames):
files = []
dids = []
for filename in filenames:
size = os.stat(str(filename)).st_size
adler = adler32(str(filename))
files.append({
'scope': self.scope,
'name': str(filename.parts[-1]),
'bytes': size,
'adler32': adler,
'pfn': self.pfn + str(filename.parts[-1])
})
replica_client = ReplicaClient()
replica_client.add_replicas(rse=self.rse, files=files)
didclient = DIDClient()
didclient.add_files_to_dataset(self.scope, self.dataset, files)
class TestDIDClients(object):
def __init__(self):
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
def test_add_and_list_archive(self):
""" ARCHIVE (CLIENT): Add files to archive and list the content."""
scope, rse = 'mock', 'MOCK'
archive_files = ['file_' + generate_uuid() + '.zip' for _ in range(2)]
files = []
for i in range(10):
files.append({
'scope': scope,
'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570,
'adler32': '0cc737eb',
'type': 'FILE',
'meta': {
'guid': str(generate_uuid())
}
})
for archive_file in archive_files:
self.replica_client.add_replicas(rse=rse,
files=[{
'scope': scope,
'name': archive_file,
'bytes': 1,
'adler32': '0cc737eb'
}])
self.did_client.add_files_to_archive(scope=scope,
name=archive_file,
files=files)
content = [
fil for fil in self.did_client.list_archive_content(
scope=scope, name=archive_file)
]
assert_equal(len(content), 10)
class TestReplicaHttpRedirection:
def setup(self):
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.marker = '$> '
# get auth token
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def test_replica_http_redirection(self):
""" REPLICA (redirection): http redirection to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" -X GET %s/redirect/%s/%s''' % (self.cacert, self.token, self.host, tmp_scope, tmp_name)
exitcode, out, err = execute(cmd)
assert_in('404 Not Found', out)
# add replicas
self.replica_client.add_replicas(rse='MOCK', files=[{'scope': tmp_scope, 'name': tmp_name, 'bytes': 1L, 'adler32': '0cc737eb'}])
def __lfns2pfns_client(self, lfns):
""" Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if neccessary.
:param scope: list of DIDs
:returns: dict with scope:name as keys and PFN as value (in case of errors the Rucio exception si assigned to the key)
"""
client = ReplicaClient()
pfns = {}
lfns = [lfns] if type(lfns) == dict else lfns
for lfn in lfns:
scope = lfn['scope']
name = lfn['name']
replicas = [r for r in client.list_replicas([{'scope': scope, 'name': name}, ], schemes=[self.attributes['scheme'], ])] # schemes is used to narrow down the response message.
if len(replicas) > 1:
pfns['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('This operation can only be performed for files.')
if not len(replicas):
pfns['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('File not found.')
pfns['%s:%s' % (scope, name)] = replicas[0]['rses'][self.rse['rse']][0] if (self.rse['rse'] in replicas[0]['rses'].keys()) else exception.RSEOperationNotSupported('Replica not found on given RSE.')
return pfns
def __lfns2pfns_client(self, lfns):
""" Provides the path of a replica for non-detemernisic sites. Will be assigned to get path by the __init__ method if neccessary.
:param scope: list of DIDs
:returns: dict with scope:name as keys and PFN as value (in case of errors the Rucio exception si assigned to the key)
"""
client = ReplicaClient()
pfns = {}
lfns = [lfns] if type(lfns) == dict else lfns
for lfn in lfns:
scope = lfn['scope']
name = lfn['name']
replicas = [r for r in client.list_replicas([{'scope': scope, 'name': name}, ], schemes=[self.attributes['scheme'], ])] # schemes is used to narrow down the response message.
if len(replicas) > 1:
pfns['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('This operation can only be performed for files.')
if not len(replicas):
pfns['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('File not found.')
pfns['%s:%s' % (scope, name)] = replicas[0]['rses'][self.rse['rse']][0] if (self.rse['rse'] in replicas[0]['rses'].keys()) else exception.RSEOperationNotSupported('Replica not found on given RSE.')
return pfns
def setUpClass(cls):
cls.dataset = 'dataset_%s' % generate_uuid()
cls.rule_client = RuleClient()
cls.did_client = DIDClient()
cls.replica_client = ReplicaClient()
cls.upload_client = UploadClient()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
cls.rse_id = get_rse_id(rse=cls.rse, **cls.vo)
def __init__(self, dataset_name, start_time, end_time, frtype,
datafind_server=None, scope=None, site=None, rse=None, check=True,
lifetime=None, dry_run=False, no_multiprocs=False,
nprocs=MAXTHREADS):
if datafind_server is None:
# If undefined, use default from environment
try:
self.LIGO_DATAFIND_SERVER=os.environ['LIGO_DATAFIND_SERVER']
# If no datafind server in env, use cvmfs server
except:
self.LIGO_DATAFIND_SERVER="datafind.ligo.org:443"
else:
self.LIGO_DATAFIND_SERVER=datafind_server
self.dataset_name = dataset_name
self.start_time = start_time
self.end_time = end_time
self.frtype = frtype
self.no_multiprocs = no_multiprocs
logging.info("Attempting to determine scope from GPS time")
if scope is None:
self.scope = get_scope(start_time, end_time)
else:
self.scope=scope
logging.info("Scope: {}".format(self.scope))
self.site = site
if rse is None:
rse = site
self.rse = rse
self.check = check
self.lifetime = lifetime
self.dry_run = dry_run
# Initialization for dataset
self.get_global_url()
self.did_client = DIDClient()
self.rep_client = ReplicaClient()
# Locate frames
frames = self.find_frames()
# Create list of rucio replicas
self.list_files(frames, nprocs=nprocs)
def __init__(self, myscope, orgRse, destRse, account='bruzzese', working_folder=None):
self.myscope = myscope
self.orgRse = orgRse
self.destRse = destRse
self.working_folder = working_folder
self.gfal = Gfal2Context()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.rulesClient = RuleClient()
# Configuration
self.account = account
# account=account
self.client = Client(account=self.account)
def test_automatix(self):
""" MULTI VO (DAEMON): Test that automatix runs on a single VO """
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
shr_scope = 'shr_%s' % scope_uuid
scope_client.add_scope('root', shr_scope)
add_scope(shr_scope, 'root', 'root', **self.new_vo)
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
shr_rse = 'SHR_%s' % rse_str
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/automatix',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
rse_client.add_rse(shr_rse)
rse_client.add_rse_attribute(rse=shr_rse, key='verify_checksum', value=False)
rse_client.add_protocol(shr_rse, mock_protocol)
add_rse(shr_rse, 'root', **self.new_vo)
add_rse_attribute(rse=shr_rse, key='verify_checksum', value=False, issuer='root', **self.new_vo)
add_protocol(rse=shr_rse, data=mock_protocol, issuer='root', **self.new_vo)
automatix(sites=[shr_rse], inputfile='/opt/rucio/etc/automatix.json', sleep_time=30, account='root', once=True, scope=shr_scope)
did_list_tst = list(DIDClient().list_dids(shr_scope, {}))
did_list_new = list(list_dids(shr_scope, {}, **self.new_vo))
assert_not_equal(len(did_list_tst), 0)
assert_equal(len(did_list_new), 0)
did_dicts = [{'scope': shr_scope, 'name': n} for n in did_list_tst]
replicas_tst = list(ReplicaClient().list_replicas(did_dicts, rse_expression=shr_rse))
replicas_new = list(list_replicas(did_dicts, rse_expression=shr_rse, **self.new_vo))
assert_not_equal(len(replicas_tst), 0)
assert_equal(len(replicas_new), 0)
GError,
)
# In[2]:
# Rucio settings
## Remember to always have an alive and valid proxy before executing the script
## account user
account = 'root'
auth_type = 'x509_proxy'
CLIENT = Client(account=account, auth_type=auth_type)
didc = DIDClient(account=account, auth_type=auth_type)
repc = ReplicaClient(account=account, auth_type=auth_type)
client = Client(account=account, auth_type=auth_type)
uploadClient = UploadClient()
downloadClient = DownloadClient()
rulesClient = RuleClient()
## user scope
Default_Scope = 'test-root'
# Get list of all RSEs
default_rses = list(client.list_rses())
rses_lists = []
for single_rse in default_rses:
rses_lists.append(single_rse['rse'])
print(rses_lists)
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.replica_client = ReplicaClient()
# Using two test RSEs
self.rse4suspicious = 'MOCK_SUSPICIOUS'
self.rse4suspicious_id = get_rse_id(self.rse4suspicious, **self.vo)
self.rse4recovery = 'MOCK_RECOVERY'
self.rse4recovery_id = get_rse_id(self.rse4recovery, **self.vo)
self.scope = 'mock'
self.internal_scope = InternalScope(self.scope, **self.vo)
# For testing, we create 3 files and upload them to Rucio to two test RSEs.
self.tmp_file1 = file_generator()
self.tmp_file2 = file_generator()
self.tmp_file3 = file_generator()
self.tmp_file4 = file_generator()
self.tmp_file5 = file_generator()
self.listdids = [{'scope': self.internal_scope, 'name': path.basename(f), 'type': DIDType.FILE}
for f in [self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5]]
for rse in [self.rse4suspicious, self.rse4recovery]:
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5} {6}'.format(rse, self.scope, self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5)
exitcode, out, err = execute(cmd)
# checking if Rucio upload went OK
assert exitcode == 0
# Set fictional datatypes
set_metadata(self.internal_scope, path.basename(self.tmp_file4), 'datatype', 'testtypedeclarebad')
set_metadata(self.internal_scope, path.basename(self.tmp_file5), 'datatype', 'testtypenopolicy')
# Allow for the RSEs to be affected by the suspicious file recovery daemon
add_rse_attribute(self.rse4suspicious_id, "enable_suspicious_file_recovery", True)
add_rse_attribute(self.rse4recovery_id, "enable_suspicious_file_recovery", True)
# removing physical files from /tmp location - keeping only their DB info
remove(self.tmp_file1)
remove(self.tmp_file2)
remove(self.tmp_file3)
remove(self.tmp_file4)
remove(self.tmp_file5)
# Gather replica info
replicalist = list_replicas(dids=self.listdids)
# Changing the replica statuses as follows:
# ----------------------------------------------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype"
# ----------------------------------------------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available) RAW
# tmp_file4 unavailable suspicious (available) testtypedeclarebad
# tmp_file5 unavailable suspicious (available) testtypenopolicy
# ----------------------------------------------------------------------------------------------------------------------------------
for replica in replicalist:
suspicious_pfns = replica['rses'][self.rse4suspicious_id]
for i in range(3):
print("Declaring suspicious file replica: " + suspicious_pfns[0])
self.replica_client.declare_suspicious_file_replicas([suspicious_pfns[0], ], 'This is a good reason.')
sleep(1)
if replica['name'] == path.basename(self.tmp_file2):
print("Declaring bad file replica: " + suspicious_pfns[0])
self.replica_client.declare_bad_file_replicas([suspicious_pfns[0], ], 'This is a good reason')
if replica['name'] == path.basename(self.tmp_file3):
print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file3), ReplicaState.UNAVAILABLE)
if replica['name'] == path.basename(self.tmp_file4):
print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file4), ReplicaState.UNAVAILABLE)
if replica['name'] == path.basename(self.tmp_file5):
print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file5), ReplicaState.UNAVAILABLE)
# Gather replica info after setting initial replica statuses
replicalist = list_replicas(dids=self.listdids)
# Checking if the status changes were effective
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file2):
assert (self.rse4suspicious_id in replica['states']) is False
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file3):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
if replica['name'] == path.basename(self.tmp_file4):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
if replica['name'] == path.basename(self.tmp_file5):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
# Checking if only self.tmp_file2 is declared as 'BAD'
self.from_date = datetime.now() - timedelta(days=1)
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
"""
Register Exist Files
Take a list of dictionaries of files that exist on SE,
add to Rucio DB.
Should be taking a list of JSON files instead?
Ex: python ~/rucyio/file_transfer/register_existfile.py --scope ams-2014-ISS.B950-pass6-sada --se-acrym tw-eos01 --rse TW-EOS01_NONDET_AMS02SCRATCHDISK ~/chchao/rucyio/file_transfer/filelists/tw-eos01-pass6-haino-root-output
"""
import argparse
import os
# from rucio.common.utils import generate_uuid
from rucio.client.replicaclient import ReplicaClient
repCli = ReplicaClient()
import sys
from rucio.client.didclient import DIDClient
didCli = DIDClient()
from rucio.client.ruleclient import RuleClient
ruleCli = RuleClient()
from rucio.client.rseclient import RSEClient
rseCli = RSEClient()
from rucio.common import exception
from filelist_comparator import eos_find2dict
from rucio.common.utils import generate_uuid
if __name__ == '__main__':
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]), add_help=True, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('Filename', metavar='Filelist', type=str, help='Filelist from eos find on target SE')
parser.add_argument('--rse', metavar='rse', type=str, help='Target RSE of the filelist, e.g:TW-EOS01_AMS02DATADISK')
parser.add_argument('--scope', '-s', metavar='Scope', type=str, help='Scope to regisister the files; e.g:ams-2014-ISS.B950-pass6')
parser.add_argument('--se', metavar='SE', type=str, help='hostname for RSE; e.g:tw-eos01.grid.sinica.edu.tw')
####from client####
from rucio.client.replicaclient import ReplicaClient
rep = ReplicaClient()
#did = 'ams-user-chenghsi:Acceptance_Form.jpg'.split(':')
did = 'ams-2011B-ISS.B620-pass4:1368923945.00000001.root'
#did = 'ams-2011B-ISS.B620-pass4:2011-06-14'
did_list = did.split(':')
scope = did_list[0]
filename = did_list[1]
rse_name = 'TW-EOS01_AMS02DATADISK'
adler32 = ''
md5 = ''
bytes = 0
#print 'before:'
for x in rep.list_replicas([{'scope': scope, 'name': filename}]):
adler32 = x['adler32']
md5 = x['md5']
bytes = x['bytes']
print adler32, md5, bytes
#from rucio.client.didclient import DIDClient
#did = DIDClient()
#file_meta = did.get_metadata(scope, filename)
#rep.delete_replicas(rse_name, [{'scope': scope, 'name': filename}])
#print 'after deletion:'
#for x in rep.list_replicas([{'scope': scope, 'name': filename}]):
# print x
#rep.add_replica(rse_name, scope, filename, bytes, adler32, md5, file_meta)
print 'test'
#print 'after add:'
#for x in rep.list_replicas([{'scope': scope, 'name': filename}]):
# print x
####from client####
from rucio.client.replicaclient import ReplicaClient
repCli = ReplicaClient()
#did = 'ams-user-chenghsi:Acceptance_Form.jpg'.split(':')
#did = 'ams-2011B-ISS.B620-pass4:1368923945.00000001.root'
import sys
from rucio.client.didclient import DIDClient
didCli = DIDClient()
from rucio.client.ruleclient import RuleClient
ruleCli = RuleClient()
from rucio.client.rseclient import RSEClient
rseCli = RSEClient()
from rucio.common import exception
argv_file =str(sys.argv[1])
with open(argv_file, 'r') as dids:
for did in dids:
did = did.rstrip('\n')
print did
if ':' in did:
did_list = did.split(':')
scope = did_list[0]
filename = did_list[1]
else:
filename = did
scope = 'ams-user-testuser1'
rse_name = 'TW-EOS00_AMS02DATADISK'
adler32 = ''
md5 = ''
bytes = 0
#print 'before:'
for x in repCli.list_replicas([{'scope': scope, 'name': filename}]):
class TestDIDClients:
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def test_list_dids(self):
""" DATA IDENTIFIERS (CLIENT): List dids by pattern."""
tmp_scope = scope_name_generator()
tmp_files = []
tmp_files.append('file_a_1%s' % generate_uuid())
tmp_files.append('file_a_2%s' % generate_uuid())
tmp_files.append('file_b_1%s' % generate_uuid())
tmp_rse = 'MOCK'
self.scope_client.add_scope('jdoe', tmp_scope)
for tmp_file in tmp_files:
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L, '0cc737eb')
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\_a\_*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\_a\_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 1)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\__\_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='file'):
results.append(result)
assert_equal(len(results), 3)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}):
results.append(result)
assert_equal(len(results), 0)
with assert_raises(UnsupportedOperation):
self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='whateverytype')
def test_list_by_metadata(self):
""" DATA IDENTIFIERS (CLIENT): List did with metadata"""
dsns = []
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn1)
dataset_meta = {'project': 'data12_8TeV',
'run_number': 400000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m920',
}
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn1, meta=dataset_meta)
tmp_dsn2 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn2)
dataset_meta['run_number'] = 400001
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn2, meta=dataset_meta)
tmp_dsn3 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn3)
dataset_meta['stream_name'] = 'physics_Egamma'
dataset_meta['datatype'] = 'NTUP_SMWZ'
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn3, meta=dataset_meta)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'version': 'f392_m920'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn1)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'run_number': 400001})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn2)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'stream_name': 'physics_Egamma', 'datatype': 'NTUP_SMWZ'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
with assert_raises(KeyNotFound):
self.did_client.list_dids(tmp_scope, {'NotReallyAKey': 'NotReallyAValue'})
def test_add_did(self):
""" DATA IDENTIFIERS (CLIENT): Add, populate and list did content"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
tmp_dsn = 'dsn_%s' % generate_uuid()
# PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename
dataset_meta = {'project': 'data13_hip',
'run_number': 300000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m927',
}
rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}]
with assert_raises(ScopeNotFound):
self.did_client.add_dataset(scope='Nimportnawak', name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
with assert_raises(DataIdentifierNotFound):
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=[{'scope': tmp_scope, 'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()),
'bytes': 724963570L, 'adler32': '0cc737eb'}, ])
files = []
for i in xrange(5):
lfn = 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 10}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
files = []
for i in xrange(5):
lfn = '%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 100}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
self.did_client.close(scope=tmp_scope, name=tmp_dsn)
def test_attach_dids_to_dids(self):
""" DATA IDENTIFIERS (CLIENT): Attach dids to dids"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
nb_datasets = 5
nb_files = 5
attachments, dsns = list(), list()
guid_to_query = None
dsn = {}
for i in xrange(nb_datasets):
attachment = {}
attachment['scope'] = tmp_scope
attachment['name'] = 'dsn.%s' % str(generate_uuid())
attachment['rse'] = tmp_rse
files = []
for i in xrange(nb_files):
files.append({'scope': tmp_scope, 'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570L, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}})
attachment['dids'] = files
guid_to_query = files[0]['meta']['guid']
dsn = {'scope': tmp_scope, 'name': attachment['name']}
dsns.append(dsn)
attachments.append(attachment)
self.did_client.add_datasets(dsns=dsns)
self.did_client.attach_dids_to_dids(attachments=attachments)
l = [i for i in self.did_client.get_dataset_by_guid(guid_to_query)]
assert_equal([dsn], l)
cnt_name = 'cnt_%s' % generate_uuid()
self.did_client.add_container(scope='mock', name=cnt_name)
with assert_raises(UnsupportedOperation):
self.did_client.attach_dids_to_dids([{'scope': 'mock', 'name': cnt_name, 'rse': tmp_rse, 'dids': attachment['dids']}])
def test_add_dataset(self):
""" DATA IDENTIFIERS (CLIENT): Add dataset """
tmp_scope = 'mock'
tmp_dsn = 'dsn_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, meta={'project': 'data13_hip'})
did = self.did_client.get_did(tmp_scope, tmp_dsn)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_dsn)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_add_datasets(self):
""" DATA IDENTIFIERS (CLIENT): Bulk add datasets """
tmp_scope = 'mock'
dsns = list()
for i in xrange(500):
tmp_dsn = {'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'meta': {'project': 'data13_hip'}}
dsns.append(tmp_dsn)
self.did_client.add_datasets(dsns)
def test_exists(self):
""" DATA IDENTIFIERS (CLIENT): Check if data identifier exists """
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
tmp_rse = 'MOCK'
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L, '0cc737eb')
did = self.did_client.get_did(tmp_scope, tmp_file)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_file)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_did_hierarchy(self):
""" DATA IDENTIFIERS (CLIENT): Check did hierarchy rule """
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(4)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(4):
self.did_client.add_did(scope, dst[i], 'DATASET', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_did(scope, cnt[i], 'CONTAINER', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1L, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1L, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': cnt[2]}, {'scope': scope, 'name': cnt[3]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
pass
# TODO: fix, fix, fix
# if r['name'] == cnt[1]:
# assert_equal(r['type'], 'container')
# assert_equal(r['level'], 0)
# if (r['name'] == cnt[0]) or (r['name'] == dst[0]) or (r['name'] == file[8]) or (r['name'] == file[9]):
# assert_equal(r['level'], 0)
# else:
# assert_equal(r['level'], 1)
def test_detach_did(self):
""" DATA IDENTIFIERS (CLIENT): Detach dids from a did"""
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(2)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1L, '0cc737eb')
for i in range(4):
self.did_client.add_dataset(scope, dst[i], statuses=None, meta=None, rules=None)
for i in range(2):
self.did_client.add_container(scope, cnt[i], statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1L, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1L, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': dst[2]}, {'scope': scope, 'name': dst[3]}])
with assert_raises(UnsupportedOperation):
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': cnt[1]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
self.did_client.detach_dids(scope, cnt[0], [{'scope': scope, 'name': dst[1]}])
self.did_client.detach_dids(scope, dst[3], [{'scope': scope, 'name': file[6]}, {'scope': scope, 'name': file[7]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
if r['name'] == dst[1]:
assert_equal(r['level'], 0)
if r['type'] is 'file':
if (r['name'] in file[6:9]):
assert_equal(r['level'], 0)
else:
assert_not_equal(r['level'], 0)
with assert_raises(UnsupportedOperation):
self.did_client.detach_dids(scope=scope, name=cnt[0], dids=[{'scope': scope, 'name': cnt[0]}])
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in xrange(3)]
self.tmp_scopes = [scope_name_generator() for i in xrange(3)]
self.tmp_rses = [rse_name_generator() for i in xrange(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in xrange(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in xrange(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in xrange(3)]
# add dummy data to the catalogue
for i in xrange(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1L, '0cc737eb')
# put files in datasets
for i in xrange(3):
for j in xrange(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1L, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
