def test_add_files(self):
""" DIRAC (CLIENT): Add a list of files."""
lfn = '/belle/MC/bnl/release-06-00-08/DB00000000/MC13/prod00000002/s00/e0000/4S/r00000/1310040140/mdst/sub00/myfile.root'
lfns = {lfn: {'rse': self.rse, 'bytes': 1234, 'adler32': 'AB1463EF', 'pfn': self.prefix + lfn}}
lfns = []
for cnt in range(30):
lfns.append({'rse': 'Mock', 'bytes': 1234, 'adler32': 'AB1463EF', 'pfn': '%s%s.%s' % (self.prefix, lfn, cnt), 'lfn': '%s.%s' % (lfn, cnt)})
self.client.add_files(lfns=lfns)
lfn_split = lfn.split('/')
lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))]
idx = 1
for lpn in lpns:
scope, name = extract_scope(lpn)
content = [str(did['name']) for did in self.client.list_content(scope, name)]
if idx < len(lpns):
print(content)
assert lpns[idx] in content
idx += 1
dsn = "/".join(lfn_split[:-1])
scope, name = extract_scope(dsn)
files = [str(did['name']) for did in self.client.list_files(scope, name)]
files.sort()
list_lfns = [str(did['lfn']) for did in lfns]
list_lfns.sort()
assert files == list_lfns
files = [str(did['rses'][self.rse][0]) for did in self.client.list_replicas([{'scope': scope, 'name': name}], schemes=['srm'])]
files.sort()
print(files)
list_pfns = [str(did['pfn']) for did in lfns]
list_pfns.sort()
assert files == list_pfns
def belleii_bootstrap(client):
scopes = [
'raw', 'hraw', 'other', 'mc_tmp', 'mc', 'test', 'user', 'data',
'data_tmp', 'group'
]
for scope in scopes:
try:
client.add_scope(scope=scope, account='root')
except Duplicate:
pass
except Exception as err:
print(err)
lpns = ['/belle', '/belle/mc', '/belle/Data', '/belle/user', '/belle/raw']
for lpn in lpns:
scope, name = extract_scope(lpn)
try:
client.add_did(scope=scope, name=name, did_type='CONTAINER')
except Duplicate:
pass
except Exception as err:
print(err)
if name != '/belle':
try:
client.attach_dids(scope='other',
name='/belle',
dids=[{
'scope': str(scope),
'name': str(name)
}])
except DuplicateContent:
pass
except Exception as err:
print(err)
def test_belle2_schema(rse_factory, did_factory, root_account, did_client):
""" BELLE2 SCHEMA (COMMON): Basic tests on Belle II schema """
bad_paths = ['invalid_name', '/belle/invalid@/did']
for path in bad_paths:
scope, name = extract_scope(path, [])
with pytest.raises(InvalidObject):
validate_schema('did', {'name': name, 'scope': scope, 'type': 'CONTAINER'})
def test_dirac_addfile(rse_factory, did_factory, root_account, did_client, dirac_client, rse_client, replica_client):
""" DIRAC (CLIENT): Test the functionality of the addfile method """
nbfiles = 5
rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True)
rse_client.add_rse_attribute(rse=rse1, key='ANY', value='True')
# Create replicas on rse1 using addfile
lfns = [{'lfn': did_name_generator('file'), 'rse': rse1, 'bytes': 1, 'adler32': '0cc737eb', 'guid': generate_uuid()} for _ in range(nbfiles)]
files = [{'scope': extract_scope(lfn['lfn'], [])[0], 'name': lfn['lfn']} for lfn in lfns]
reps = [{'scope': extract_scope(lfn['lfn'], [])[0], 'name': lfn['lfn'], 'rse': rse1} for lfn in lfns]
dirac_client.add_files(lfns=lfns, ignore_availability=False)
replicas = [rep for rep in replica_client.list_replicas(dids=files)]
for replica in replicas:
assert {'scope': replica['scope'], 'name': replica['name'], 'rse': list(replica['rses'].keys())[0]} in reps
# Check the existence of all parents from the files
for lfn in lfns:
directories = lfn['lfn'].split('/')
for cnt, directory in enumerate(directories):
parent = "/".join(directories[0:cnt])
child = "/".join(directories[0:cnt + 1])
if parent != '':
parent_scope, parent_name = extract_scope(parent, [])
children = [did['name'] for did in did_client.list_content(parent_scope, parent_name)]
assert child in children
# Check that the default rules are created
for lfn in lfns:
# Check dataset rule
directory = "/".join(lfn['lfn'].split('/')[:-1])
scope, name = extract_scope(directory, [])
rules = [rule for rule in did_client.list_did_rules(scope, name)]
assert len(rules) == 1
assert rules[0]['rse_expression'] == 'ANY=true'
assert rules[0]['expires_at'] is None
# Check file rule
scope, name = extract_scope(lfn['lfn'], [])
rules = [rule for rule in did_client.list_did_rules(scope, name)]
assert len(rules) == 1
assert rules[0]['rse_expression'] == rse1
assert datetime.now() - rules[0]['expires_at'] < timedelta(hours=24)
def add_files(lfns, issuer, ignore_availability, vo='def'):
"""
Bulk add files :
- Create the file and replica.
- If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites.
- Create all the ascendants of the dataset if they do not exist
:param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
:param issuer: The issuer account.
:param ignore_availability: A boolean to ignore blocked sites.
:param vo: The VO to act on.
"""
scopes = list_scopes(vo=vo)
dids = []
rses = {}
for lfn in lfns:
scope, name = extract_scope(lfn['lfn'], scopes)
dids.append({'scope': scope, 'name': name})
rse = lfn['rse']
if rse not in rses:
rse_id = get_rse_id(rse=rse, vo=vo)
rses[rse] = rse_id
lfn['rse_id'] = rses[rse]
# Check if the issuer can add dids and use skip_availabitlity
for rse in rses:
rse_id = rses[rse]
kwargs = {'rse': rse, 'rse_id': rse_id}
if not has_permission(
issuer=issuer, action='add_replicas', kwargs=kwargs, vo=vo):
raise AccessDenied(
'Account %s can not add file replicas on %s for VO %s' %
(issuer, rse, vo))
if not has_permission(issuer=issuer,
action='skip_availability_check',
kwargs=kwargs,
vo=vo):
ignore_availability = False
# Check if the issuer can add the files
kwargs = {'issuer': issuer, 'dids': dids}
if not has_permission(
issuer=issuer, action='add_dids', kwargs=kwargs, vo=vo):
raise AccessDenied(
'Account %s can not bulk add data identifier for VO %s' %
(issuer, vo))
dirac.add_files(lfns=lfns,
account=issuer,
ignore_availability=ignore_availability,
session=None,
vo=vo)
def get_scope(lfn, scopes=None, diracAlgorithm="dirac"):
"""
Helper function that extracts the scope from the LFN.
:param str lfn: Logical file name
:param list scopes: list of scopes
:param str diracAlgorithm: only used by extract_scope if there is no config file with an algorithm listed.
Otherwise use the algorithm listed in the config file.
:return: scope name
"""
if scopes is None:
scopes = []
scope, _ = extract_scope(did=lfn,
scopes=scopes,
default_extract=diracAlgorithm)
return scope
def add_files(lfns, account, ignore_availability, session=None):
"""
Bulk add files :
- Create the file and replica.
- If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites.
- Create all the ascendants of the dataset if they do not exist
:param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
:param issuer: The issuer account.
:param ignore_availability: A boolean to ignore blocklisted sites.
:session: The session used
"""
attachments = []
# The list of scopes is necessary for the extract_scope
scopes = list_scopes(session=session)
scopes = [scope.external for scope in scopes]
exist_lfn = []
for lfn in lfns:
# First check if the file exists
filename = lfn['lfn']
lfn_scope, _ = extract_scope(filename, scopes)
lfn_scope = InternalScope(lfn_scope)
exists, did_type = _exists(lfn_scope, filename)
if exists:
continue
# Get all the ascendants of the file
lfn_split = filename.split('/')
lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))]
lpns.reverse()
print(lpns)
# The parent must be a dataset. Register it as well as the rule
dsn_name = lpns[0]
dsn_scope, _ = extract_scope(dsn_name, scopes)
dsn_scope = InternalScope(dsn_scope)
exists, did_type = _exists(dsn_scope, dsn_name)
if exists and did_type == DIDType.CONTAINER:
raise UnsupportedOperation('Cannot create %s as dataset' %
dsn_name)
if (dsn_name not in exist_lfn) and not exists:
print('Will create %s' % dsn_name)
add_did(scope=dsn_scope,
name=dsn_name,
type=DIDType.DATASET,
account=InternalAccount(account),
statuses=None,
meta=None,
rules=[{
'copies': 1,
'rse_expression': 'ANY=true',
'weight': None,
'account': InternalAccount(account),
'lifetime': None,
'grouping': 'NONE'
}],
lifetime=None,
dids=None,
rse_id=None,
session=session)
exist_lfn.append(dsn_name)
parent_name = lpns[1]
parent_scope, _ = extract_scope(parent_name, scopes)
parent_scope = InternalScope(parent_scope)
attachments.append({
'scope': parent_scope,
'name': parent_name,
'dids': [{
'scope': dsn_scope,
'name': dsn_name
}]
})
# Register the file
rse_id = lfn.get('rse_id', None)
if not rse_id:
raise InvalidType('Missing rse_id')
bytes = lfn.get('bytes', None)
guid = lfn.get('guid', None)
adler32 = lfn.get('adler32', None)
pfn = lfn.get('pfn', None)
files = {
'scope': lfn_scope,
'name': filename,
'bytes': bytes,
'adler32': adler32
}
if pfn:
files['pfn'] = str(pfn)
if guid:
files['meta'] = {'guid': guid}
add_replicas(rse_id=rse_id,
files=[files],
dataset_meta=None,
account=InternalAccount(account),
ignore_availability=ignore_availability,
session=session)
add_rule(dids=[{
'scope': lfn_scope,
'name': filename
}],
account=InternalAccount(account),
copies=1,
rse_expression=lfn['rse'],
grouping=None,
weight=None,
lifetime=86400,
locked=None,
subscription_id=None,
session=session)
attachments.append({
'scope': dsn_scope,
'name': dsn_name,
'dids': [{
'scope': lfn_scope,
'name': filename
}]
})
# Now loop over the ascendants of the dataset and created them
for lpn in lpns[1:]:
child_scope, _ = extract_scope(lpn, scopes)
child_scope = InternalScope(child_scope)
exists, did_type = _exists(child_scope, lpn)
if exists and did_type == DIDType.DATASET:
raise UnsupportedOperation('Cannot create %s as container' %
lpn)
if (lpn not in exist_lfn) and not exists:
print('Will create %s' % lpn)
add_did(scope=child_scope,
name=lpn,
type=DIDType.CONTAINER,
account=InternalAccount(account),
statuses=None,
meta=None,
rules=None,
lifetime=None,
dids=None,
rse_id=None,
session=session)
exist_lfn.append(lpn)
parent_name = lpns[lpns.index(lpn) + 1]
parent_scope, _ = extract_scope(parent_name, scopes)
parent_scope = InternalScope(parent_scope)
attachments.append({
'scope':
parent_scope,
'name':
parent_name,
'dids': [{
'scope': child_scope,
'name': lpn
}]
})
# Finally attach everything
attach_dids_to_dids(attachments,
account=InternalAccount(account),
ignore_duplicate=True,
session=session)
def add_files(lfns, account, ignore_availability, vo='def', session=None):
"""
Bulk add files :
- Create the file and replica.
- If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites.
- Create all the ascendants of the dataset if they do not exist
:param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
:param issuer: The issuer account.
:param ignore_availability: A boolean to ignore blocklisted sites.
:param vo: The VO to act on
:param session: The session used
"""
rule_extension_list = []
attachments = []
# The list of scopes is necessary for the extract_scope
filter_ = {'scope': InternalScope(scope='*', vo=vo)}
scopes = list_scopes(filter_=filter_, session=session)
scopes = [scope.external for scope in scopes]
exist_lfn = []
try:
lifetime_dict = config_get(section='dirac', option='lifetime', session=session)
lifetime_dict = loads(lifetime_dict)
except ConfigNotFound:
lifetime_dict = {}
except JSONDecodeError as err:
raise InvalidType('Problem parsing lifetime option in dirac section : %s' % str(err))
except Exception as err:
raise RucioException(str(err))
for lfn in lfns:
# First check if the file exists
filename = lfn['lfn']
lfn_scope, _ = extract_scope(filename, scopes)
lfn_scope = InternalScope(lfn_scope, vo=vo)
exists, did_type = _exists(lfn_scope, filename)
if exists:
continue
# Get all the ascendants of the file
lfn_split = filename.split('/')
lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))]
lpns.reverse()
print(lpns)
# The parent must be a dataset. Register it as well as the rule
dsn_name = lpns[0]
dsn_scope, _ = extract_scope(dsn_name, scopes)
dsn_scope = InternalScope(dsn_scope, vo=vo)
# Compute lifetime
lifetime = None
if dsn_scope in lifetime_dict:
lifetime = lifetime_dict[dsn_scope]
else:
for pattern in lifetime_dict:
if re.match(pattern, dsn_scope):
lifetime = lifetime_dict[pattern]
break
exists, did_type = _exists(dsn_scope, dsn_name)
if exists and did_type == DIDType.CONTAINER:
raise UnsupportedOperation('Cannot create %s as dataset' % dsn_name)
if (dsn_name not in exist_lfn) and not exists:
print('Will create %s' % dsn_name)
# to maintain a compatibility between master and LTS-1.26 branches remove keywords for first 3 arguments
add_did(dsn_scope,
dsn_name,
DIDType.DATASET,
account=InternalAccount(account, vo=vo),
statuses=None,
meta=None,
rules=[{'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account, vo=vo), 'lifetime': None, 'grouping': 'NONE'}],
lifetime=None,
dids=None,
rse_id=None,
session=session)
exist_lfn.append(dsn_name)
parent_name = lpns[1]
parent_scope, _ = extract_scope(parent_name, scopes)
parent_scope = InternalScope(parent_scope, vo=vo)
attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': dsn_scope, 'name': dsn_name}]})
rule_extension_list.append((dsn_scope, dsn_name))
if lifetime and (dsn_scope, dsn_name) not in rule_extension_list:
# Reset the lifetime of the rule to the configured value
rule = [rul for rul in list_rules({'scope': dsn_scope, 'name': dsn_name, 'account': InternalAccount(account, vo=vo)}, session=session) if rul['rse_expression'] == 'ANY=true']
if rule:
update_rule(rule[0]['id'], options={'lifetime': lifetime}, session=session)
rule_extension_list.append((dsn_scope, dsn_name))
# Register the file
rse_id = lfn.get('rse_id', None)
if not rse_id:
raise InvalidType('Missing rse_id')
bytes_ = lfn.get('bytes', None)
guid = lfn.get('guid', None)
adler32 = lfn.get('adler32', None)
pfn = lfn.get('pfn', None)
files = {'scope': lfn_scope, 'name': filename, 'bytes': bytes_, 'adler32': adler32}
if pfn:
files['pfn'] = str(pfn)
if guid:
files['meta'] = {'guid': guid}
add_replicas(rse_id=rse_id,
files=[files],
dataset_meta=None,
account=InternalAccount(account, vo=vo),
ignore_availability=ignore_availability,
session=session)
add_rule(dids=[{'scope': lfn_scope, 'name': filename}],
account=InternalAccount(account, vo=vo),
copies=1,
rse_expression=lfn['rse'],
grouping=None,
weight=None,
lifetime=86400,
locked=None,
subscription_id=None,
session=session)
attachments.append({'scope': dsn_scope, 'name': dsn_name, 'dids': [{'scope': lfn_scope, 'name': filename}]})
# Now loop over the ascendants of the dataset and created them
for lpn in lpns[1:]:
child_scope, _ = extract_scope(lpn, scopes)
child_scope = InternalScope(child_scope, vo=vo)
exists, did_type = _exists(child_scope, lpn)
if exists and did_type == DIDType.DATASET:
raise UnsupportedOperation('Cannot create %s as container' % lpn)
if (lpn not in exist_lfn) and not exists:
print('Will create %s' % lpn)
add_did(child_scope,
lpn,
DIDType.CONTAINER,
account=InternalAccount(account, vo=vo),
statuses=None,
meta=None,
rules=None,
lifetime=None,
dids=None,
rse_id=None,
session=session)
exist_lfn.append(lpn)
parent_name = lpns[lpns.index(lpn) + 1]
parent_scope, _ = extract_scope(parent_name, scopes)
parent_scope = InternalScope(parent_scope, vo=vo)
attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': child_scope, 'name': lpn}]})
# Finally attach everything
attach_dids_to_dids(attachments,
account=InternalAccount(account, vo=vo),
ignore_duplicate=True,
session=session)