def test_add_files(self): """ DIRAC (CLIENT): Add a list of files.""" lfn = '/belle/MC/bnl/release-06-00-08/DB00000000/MC13/prod00000002/s00/e0000/4S/r00000/1310040140/mdst/sub00/myfile.root' lfns = {lfn: {'rse': self.rse, 'bytes': 1234, 'adler32': 'AB1463EF', 'pfn': self.prefix + lfn}} lfns = [] for cnt in range(30): lfns.append({'rse': 'Mock', 'bytes': 1234, 'adler32': 'AB1463EF', 'pfn': '%s%s.%s' % (self.prefix, lfn, cnt), 'lfn': '%s.%s' % (lfn, cnt)}) self.client.add_files(lfns=lfns) lfn_split = lfn.split('/') lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))] idx = 1 for lpn in lpns: scope, name = extract_scope(lpn) content = [str(did['name']) for did in self.client.list_content(scope, name)] if idx < len(lpns): print(content) assert lpns[idx] in content idx += 1 dsn = "/".join(lfn_split[:-1]) scope, name = extract_scope(dsn) files = [str(did['name']) for did in self.client.list_files(scope, name)] files.sort() list_lfns = [str(did['lfn']) for did in lfns] list_lfns.sort() assert files == list_lfns files = [str(did['rses'][self.rse][0]) for did in self.client.list_replicas([{'scope': scope, 'name': name}], schemes=['srm'])] files.sort() print(files) list_pfns = [str(did['pfn']) for did in lfns] list_pfns.sort() assert files == list_pfns
def belleii_bootstrap(client): scopes = [ 'raw', 'hraw', 'other', 'mc_tmp', 'mc', 'test', 'user', 'data', 'data_tmp', 'group' ] for scope in scopes: try: client.add_scope(scope=scope, account='root') except Duplicate: pass except Exception as err: print(err) lpns = ['/belle', '/belle/mc', '/belle/Data', '/belle/user', '/belle/raw'] for lpn in lpns: scope, name = extract_scope(lpn) try: client.add_did(scope=scope, name=name, did_type='CONTAINER') except Duplicate: pass except Exception as err: print(err) if name != '/belle': try: client.attach_dids(scope='other', name='/belle', dids=[{ 'scope': str(scope), 'name': str(name) }]) except DuplicateContent: pass except Exception as err: print(err)
def test_belle2_schema(rse_factory, did_factory, root_account, did_client): """ BELLE2 SCHEMA (COMMON): Basic tests on Belle II schema """ bad_paths = ['invalid_name', '/belle/invalid@/did'] for path in bad_paths: scope, name = extract_scope(path, []) with pytest.raises(InvalidObject): validate_schema('did', {'name': name, 'scope': scope, 'type': 'CONTAINER'})
def test_dirac_addfile(rse_factory, did_factory, root_account, did_client, dirac_client, rse_client, replica_client): """ DIRAC (CLIENT): Test the functionality of the addfile method """ nbfiles = 5 rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True) rse_client.add_rse_attribute(rse=rse1, key='ANY', value='True') # Create replicas on rse1 using addfile lfns = [{'lfn': did_name_generator('file'), 'rse': rse1, 'bytes': 1, 'adler32': '0cc737eb', 'guid': generate_uuid()} for _ in range(nbfiles)] files = [{'scope': extract_scope(lfn['lfn'], [])[0], 'name': lfn['lfn']} for lfn in lfns] reps = [{'scope': extract_scope(lfn['lfn'], [])[0], 'name': lfn['lfn'], 'rse': rse1} for lfn in lfns] dirac_client.add_files(lfns=lfns, ignore_availability=False) replicas = [rep for rep in replica_client.list_replicas(dids=files)] for replica in replicas: assert {'scope': replica['scope'], 'name': replica['name'], 'rse': list(replica['rses'].keys())[0]} in reps # Check the existence of all parents from the files for lfn in lfns: directories = lfn['lfn'].split('/') for cnt, directory in enumerate(directories): parent = "/".join(directories[0:cnt]) child = "/".join(directories[0:cnt + 1]) if parent != '': parent_scope, parent_name = extract_scope(parent, []) children = [did['name'] for did in did_client.list_content(parent_scope, parent_name)] assert child in children # Check that the default rules are created for lfn in lfns: # Check dataset rule directory = "/".join(lfn['lfn'].split('/')[:-1]) scope, name = extract_scope(directory, []) rules = [rule for rule in did_client.list_did_rules(scope, name)] assert len(rules) == 1 assert rules[0]['rse_expression'] == 'ANY=true' assert rules[0]['expires_at'] is None # Check file rule scope, name = extract_scope(lfn['lfn'], []) rules = [rule for rule in did_client.list_did_rules(scope, name)] assert len(rules) == 1 assert rules[0]['rse_expression'] == rse1 assert datetime.now() - rules[0]['expires_at'] < timedelta(hours=24)
def add_files(lfns, issuer, ignore_availability, vo='def'): """ Bulk add files : - Create the file and replica. - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. - Create all the ascendants of the dataset if they do not exist :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} :param issuer: The issuer account. :param ignore_availability: A boolean to ignore blocked sites. :param vo: The VO to act on. """ scopes = list_scopes(vo=vo) dids = [] rses = {} for lfn in lfns: scope, name = extract_scope(lfn['lfn'], scopes) dids.append({'scope': scope, 'name': name}) rse = lfn['rse'] if rse not in rses: rse_id = get_rse_id(rse=rse, vo=vo) rses[rse] = rse_id lfn['rse_id'] = rses[rse] # Check if the issuer can add dids and use skip_availabitlity for rse in rses: rse_id = rses[rse] kwargs = {'rse': rse, 'rse_id': rse_id} if not has_permission( issuer=issuer, action='add_replicas', kwargs=kwargs, vo=vo): raise AccessDenied( 'Account %s can not add file replicas on %s for VO %s' % (issuer, rse, vo)) if not has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs, vo=vo): ignore_availability = False # Check if the issuer can add the files kwargs = {'issuer': issuer, 'dids': dids} if not has_permission( issuer=issuer, action='add_dids', kwargs=kwargs, vo=vo): raise AccessDenied( 'Account %s can not bulk add data identifier for VO %s' % (issuer, vo)) dirac.add_files(lfns=lfns, account=issuer, ignore_availability=ignore_availability, session=None, vo=vo)
def get_scope(lfn, scopes=None, diracAlgorithm="dirac"): """ Helper function that extracts the scope from the LFN. :param str lfn: Logical file name :param list scopes: list of scopes :param str diracAlgorithm: only used by extract_scope if there is no config file with an algorithm listed. Otherwise use the algorithm listed in the config file. :return: scope name """ if scopes is None: scopes = [] scope, _ = extract_scope(did=lfn, scopes=scopes, default_extract=diracAlgorithm) return scope
def add_files(lfns, account, ignore_availability, session=None): """ Bulk add files : - Create the file and replica. - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. - Create all the ascendants of the dataset if they do not exist :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} :param issuer: The issuer account. :param ignore_availability: A boolean to ignore blocklisted sites. :session: The session used """ attachments = [] # The list of scopes is necessary for the extract_scope scopes = list_scopes(session=session) scopes = [scope.external for scope in scopes] exist_lfn = [] for lfn in lfns: # First check if the file exists filename = lfn['lfn'] lfn_scope, _ = extract_scope(filename, scopes) lfn_scope = InternalScope(lfn_scope) exists, did_type = _exists(lfn_scope, filename) if exists: continue # Get all the ascendants of the file lfn_split = filename.split('/') lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))] lpns.reverse() print(lpns) # The parent must be a dataset. Register it as well as the rule dsn_name = lpns[0] dsn_scope, _ = extract_scope(dsn_name, scopes) dsn_scope = InternalScope(dsn_scope) exists, did_type = _exists(dsn_scope, dsn_name) if exists and did_type == DIDType.CONTAINER: raise UnsupportedOperation('Cannot create %s as dataset' % dsn_name) if (dsn_name not in exist_lfn) and not exists: print('Will create %s' % dsn_name) add_did(scope=dsn_scope, name=dsn_name, type=DIDType.DATASET, account=InternalAccount(account), statuses=None, meta=None, rules=[{ 'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account), 'lifetime': None, 'grouping': 'NONE' }], lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(dsn_name) parent_name = lpns[1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope) attachments.append({ 'scope': parent_scope, 'name': parent_name, 'dids': [{ 'scope': dsn_scope, 'name': dsn_name }] }) # Register the file rse_id = lfn.get('rse_id', None) if not rse_id: raise InvalidType('Missing rse_id') bytes = lfn.get('bytes', None) guid = lfn.get('guid', None) adler32 = lfn.get('adler32', None) pfn = lfn.get('pfn', None) files = { 'scope': lfn_scope, 'name': filename, 'bytes': bytes, 'adler32': adler32 } if pfn: files['pfn'] = str(pfn) if guid: files['meta'] = {'guid': guid} add_replicas(rse_id=rse_id, files=[files], dataset_meta=None, account=InternalAccount(account), ignore_availability=ignore_availability, session=session) add_rule(dids=[{ 'scope': lfn_scope, 'name': filename }], account=InternalAccount(account), copies=1, rse_expression=lfn['rse'], grouping=None, weight=None, lifetime=86400, locked=None, subscription_id=None, session=session) attachments.append({ 'scope': dsn_scope, 'name': dsn_name, 'dids': [{ 'scope': lfn_scope, 'name': filename }] }) # Now loop over the ascendants of the dataset and created them for lpn in lpns[1:]: child_scope, _ = extract_scope(lpn, scopes) child_scope = InternalScope(child_scope) exists, did_type = _exists(child_scope, lpn) if exists and did_type == DIDType.DATASET: raise UnsupportedOperation('Cannot create %s as container' % lpn) if (lpn not in exist_lfn) and not exists: print('Will create %s' % lpn) add_did(scope=child_scope, name=lpn, type=DIDType.CONTAINER, account=InternalAccount(account), statuses=None, meta=None, rules=None, lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(lpn) parent_name = lpns[lpns.index(lpn) + 1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope) attachments.append({ 'scope': parent_scope, 'name': parent_name, 'dids': [{ 'scope': child_scope, 'name': lpn }] }) # Finally attach everything attach_dids_to_dids(attachments, account=InternalAccount(account), ignore_duplicate=True, session=session)
def add_files(lfns, account, ignore_availability, vo='def', session=None): """ Bulk add files : - Create the file and replica. - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. - Create all the ascendants of the dataset if they do not exist :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} :param issuer: The issuer account. :param ignore_availability: A boolean to ignore blocklisted sites. :param vo: The VO to act on :param session: The session used """ rule_extension_list = [] attachments = [] # The list of scopes is necessary for the extract_scope filter_ = {'scope': InternalScope(scope='*', vo=vo)} scopes = list_scopes(filter_=filter_, session=session) scopes = [scope.external for scope in scopes] exist_lfn = [] try: lifetime_dict = config_get(section='dirac', option='lifetime', session=session) lifetime_dict = loads(lifetime_dict) except ConfigNotFound: lifetime_dict = {} except JSONDecodeError as err: raise InvalidType('Problem parsing lifetime option in dirac section : %s' % str(err)) except Exception as err: raise RucioException(str(err)) for lfn in lfns: # First check if the file exists filename = lfn['lfn'] lfn_scope, _ = extract_scope(filename, scopes) lfn_scope = InternalScope(lfn_scope, vo=vo) exists, did_type = _exists(lfn_scope, filename) if exists: continue # Get all the ascendants of the file lfn_split = filename.split('/') lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))] lpns.reverse() print(lpns) # The parent must be a dataset. Register it as well as the rule dsn_name = lpns[0] dsn_scope, _ = extract_scope(dsn_name, scopes) dsn_scope = InternalScope(dsn_scope, vo=vo) # Compute lifetime lifetime = None if dsn_scope in lifetime_dict: lifetime = lifetime_dict[dsn_scope] else: for pattern in lifetime_dict: if re.match(pattern, dsn_scope): lifetime = lifetime_dict[pattern] break exists, did_type = _exists(dsn_scope, dsn_name) if exists and did_type == DIDType.CONTAINER: raise UnsupportedOperation('Cannot create %s as dataset' % dsn_name) if (dsn_name not in exist_lfn) and not exists: print('Will create %s' % dsn_name) # to maintain a compatibility between master and LTS-1.26 branches remove keywords for first 3 arguments add_did(dsn_scope, dsn_name, DIDType.DATASET, account=InternalAccount(account, vo=vo), statuses=None, meta=None, rules=[{'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account, vo=vo), 'lifetime': None, 'grouping': 'NONE'}], lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(dsn_name) parent_name = lpns[1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope, vo=vo) attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': dsn_scope, 'name': dsn_name}]}) rule_extension_list.append((dsn_scope, dsn_name)) if lifetime and (dsn_scope, dsn_name) not in rule_extension_list: # Reset the lifetime of the rule to the configured value rule = [rul for rul in list_rules({'scope': dsn_scope, 'name': dsn_name, 'account': InternalAccount(account, vo=vo)}, session=session) if rul['rse_expression'] == 'ANY=true'] if rule: update_rule(rule[0]['id'], options={'lifetime': lifetime}, session=session) rule_extension_list.append((dsn_scope, dsn_name)) # Register the file rse_id = lfn.get('rse_id', None) if not rse_id: raise InvalidType('Missing rse_id') bytes_ = lfn.get('bytes', None) guid = lfn.get('guid', None) adler32 = lfn.get('adler32', None) pfn = lfn.get('pfn', None) files = {'scope': lfn_scope, 'name': filename, 'bytes': bytes_, 'adler32': adler32} if pfn: files['pfn'] = str(pfn) if guid: files['meta'] = {'guid': guid} add_replicas(rse_id=rse_id, files=[files], dataset_meta=None, account=InternalAccount(account, vo=vo), ignore_availability=ignore_availability, session=session) add_rule(dids=[{'scope': lfn_scope, 'name': filename}], account=InternalAccount(account, vo=vo), copies=1, rse_expression=lfn['rse'], grouping=None, weight=None, lifetime=86400, locked=None, subscription_id=None, session=session) attachments.append({'scope': dsn_scope, 'name': dsn_name, 'dids': [{'scope': lfn_scope, 'name': filename}]}) # Now loop over the ascendants of the dataset and created them for lpn in lpns[1:]: child_scope, _ = extract_scope(lpn, scopes) child_scope = InternalScope(child_scope, vo=vo) exists, did_type = _exists(child_scope, lpn) if exists and did_type == DIDType.DATASET: raise UnsupportedOperation('Cannot create %s as container' % lpn) if (lpn not in exist_lfn) and not exists: print('Will create %s' % lpn) add_did(child_scope, lpn, DIDType.CONTAINER, account=InternalAccount(account, vo=vo), statuses=None, meta=None, rules=None, lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(lpn) parent_name = lpns[lpns.index(lpn) + 1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope, vo=vo) attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': child_scope, 'name': lpn}]}) # Finally attach everything attach_dids_to_dids(attachments, account=InternalAccount(account, vo=vo), ignore_duplicate=True, session=session)