def run(total_workers=1, once=False, inputfile=None, sleep_time=-1): """ Starts up the automatix threads. """ setup_logging() if rucio.db.sqla.util.is_old_db(): raise exception.DatabaseException('Database was not updated, daemon won\'t start') try: sites = [s.strip() for s in config_get('automatix', 'sites').split(',')] except (NoOptionError, NoSectionError, RuntimeError): raise Exception('Could not load sites from configuration') if not inputfile: inputfile = '/opt/rucio/etc/automatix.json' if sleep_time == -1: try: sleep_time = config_get('automatix', 'sleep_time') except (NoOptionError, NoSectionError, RuntimeError): sleep_time = 30 try: account = config_get('automatix', 'account') except (NoOptionError, NoSectionError, RuntimeError): account = 'root' try: dataset_lifetime = config_get('automatix', 'dataset_lifetime') except (NoOptionError, NoSectionError, RuntimeError): dataset_lifetime = None try: set_metadata = config_get('automatix', 'set_metadata') except (NoOptionError, NoSectionError, RuntimeError): set_metadata = False try: scope = config_get('automatix', 'scope') client = Client() filters = {'scope': InternalScope('*', vo=client.vo)} if InternalScope(scope, vo=client.vo) not in list_scopes(filter_=filters): logging.log(logging.ERROR, 'Scope %s does not exist. Exiting', scope) GRACEFUL_STOP.set() except Exception: scope = False threads = list() for worker_number in range(0, total_workers): kwargs = {'worker_number': worker_number, 'total_workers': total_workers, 'once': once, 'sites': sites, 'sleep_time': sleep_time, 'account': account, 'inputfile': inputfile, 'set_metadata': set_metadata, 'scope': scope, 'dataset_lifetime': dataset_lifetime} threads.append(threading.Thread(target=automatix, kwargs=kwargs)) [thread.start() for thread in threads] while threads[0].is_alive(): logging.log(logging.DEBUG, 'Still %i active threads', len(threads)) [thread.join(timeout=3.14) for thread in threads]
def list_scopes(): """ Lists all scopes. :returns: A list containing all scopes. """ return core_scope.list_scopes()
def list_scopes(): """ Lists all scopes. :returns: A list containing all scopes. """ return [scope.external for scope in core_scope.list_scopes()]
def run(total_workers=1, once=False, inputfile=None): """ Starts up the automatix threads. """ try: sites = [s.strip() for s in get('automatix', 'sites').split(',')] except Exception: raise Exception('Could not load sites from configuration') if not inputfile: inputfile = '/opt/rucio/etc/automatix.json' try: sleep_time = get('automatix', 'sleep_time') except Exception: sleep_time = 30 try: account = get('automatix', 'account') except Exception: account = 'root' try: dataset_lifetime = get('automatix', 'dataset_lifetime') except Exception: dataset_lifetime = None try: set_metadata = get('automatix', 'set_metadata') except Exception: set_metadata = False try: scope = get('automatix', 'scope') if InternalScope(scope) not in list_scopes(): logging.error('Scope %s does not exist. Exiting', scope) GRACEFUL_STOP.set() except Exception: scope = False threads = list() for worker_number in range(0, total_workers): kwargs = { 'worker_number': worker_number, 'total_workers': total_workers, 'once': once, 'sites': sites, 'sleep_time': sleep_time, 'account': account, 'inputfile': inputfile, 'set_metadata': set_metadata, 'scope': scope, 'dataset_lifetime': dataset_lifetime } threads.append(threading.Thread(target=automatix, kwargs=kwargs)) [thread.start() for thread in threads] while threads[0].is_alive(): logging.debug('Still %i active threads', len(threads)) [thread.join(timeout=3.14) for thread in threads]
def add_files(lfns, issuer, ignore_availability): """ Bulk add files : - Create the file and replica. - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. - Create all the ascendants of the dataset if they do not exist :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} :param issuer: The issuer account. :param ignore_availability: A boolean to ignore blacklisted sites. """ scopes = list_scopes() dids = [] rses = {} for lfn in lfns: scope, name = extract_scope(lfn['lfn'], scopes) dids.append({'scope': scope, 'name': name}) rse = lfn['rse'] if rse not in rses: rse_id = get_rse_id(rse=rse) rses[rse] = rse_id lfn['rse_id'] = rses[rse] # Check if the issuer can add dids and use skip_availabitlity for rse in rses: rse_id = rses[rse] kwargs = {'rse': rse, 'rse_id': rse_id} if not has_permission( issuer=issuer, action='add_replicas', kwargs=kwargs): raise AccessDenied('Account %s can not add file replicas on %s' % (issuer, rse)) if not has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs): ignore_availability = False # Check if the issuer can add the files kwargs = {'issuer': issuer, 'dids': dids} if not has_permission(issuer=issuer, action='add_dids', kwargs=kwargs): raise AccessDenied('Account %s can not bulk add data identifier' % (issuer)) issuer = InternalAccount(issuer) dirac.add_files(lfns=lfns, account=issuer, ignore_availability=ignore_availability, session=None)
def list_scopes(filter={}, vo='def'): """ Lists all scopes. :param filter: Dictionary of attributes by which the input data should be filtered :param vo: The VO to act on. :returns: A list containing all scopes. """ # If filter is empty, create a new dict to avoid overwriting the function's default if not filter: filter = {} if 'scope' in filter: filter['scope'] = InternalScope(scope=filter['scope'], vo=vo) else: filter['scope'] = InternalScope(scope='*', vo=vo) return [scope.external for scope in core_scope.list_scopes(filter=filter)]
def add_files(lfns, account, ignore_availability, session=None): """ Bulk add files : - Create the file and replica. - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. - Create all the ascendants of the dataset if they do not exist :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} :param issuer: The issuer account. :param ignore_availability: A boolean to ignore blocklisted sites. :session: The session used """ attachments = [] # The list of scopes is necessary for the extract_scope scopes = list_scopes(session=session) scopes = [scope.external for scope in scopes] exist_lfn = [] for lfn in lfns: # First check if the file exists filename = lfn['lfn'] lfn_scope, _ = extract_scope(filename, scopes) lfn_scope = InternalScope(lfn_scope) exists, did_type = _exists(lfn_scope, filename) if exists: continue # Get all the ascendants of the file lfn_split = filename.split('/') lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))] lpns.reverse() print(lpns) # The parent must be a dataset. Register it as well as the rule dsn_name = lpns[0] dsn_scope, _ = extract_scope(dsn_name, scopes) dsn_scope = InternalScope(dsn_scope) exists, did_type = _exists(dsn_scope, dsn_name) if exists and did_type == DIDType.CONTAINER: raise UnsupportedOperation('Cannot create %s as dataset' % dsn_name) if (dsn_name not in exist_lfn) and not exists: print('Will create %s' % dsn_name) add_did(scope=dsn_scope, name=dsn_name, type=DIDType.DATASET, account=InternalAccount(account), statuses=None, meta=None, rules=[{ 'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account), 'lifetime': None, 'grouping': 'NONE' }], lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(dsn_name) parent_name = lpns[1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope) attachments.append({ 'scope': parent_scope, 'name': parent_name, 'dids': [{ 'scope': dsn_scope, 'name': dsn_name }] }) # Register the file rse_id = lfn.get('rse_id', None) if not rse_id: raise InvalidType('Missing rse_id') bytes = lfn.get('bytes', None) guid = lfn.get('guid', None) adler32 = lfn.get('adler32', None) pfn = lfn.get('pfn', None) files = { 'scope': lfn_scope, 'name': filename, 'bytes': bytes, 'adler32': adler32 } if pfn: files['pfn'] = str(pfn) if guid: files['meta'] = {'guid': guid} add_replicas(rse_id=rse_id, files=[files], dataset_meta=None, account=InternalAccount(account), ignore_availability=ignore_availability, session=session) add_rule(dids=[{ 'scope': lfn_scope, 'name': filename }], account=InternalAccount(account), copies=1, rse_expression=lfn['rse'], grouping=None, weight=None, lifetime=86400, locked=None, subscription_id=None, session=session) attachments.append({ 'scope': dsn_scope, 'name': dsn_name, 'dids': [{ 'scope': lfn_scope, 'name': filename }] }) # Now loop over the ascendants of the dataset and created them for lpn in lpns[1:]: child_scope, _ = extract_scope(lpn, scopes) child_scope = InternalScope(child_scope) exists, did_type = _exists(child_scope, lpn) if exists and did_type == DIDType.DATASET: raise UnsupportedOperation('Cannot create %s as container' % lpn) if (lpn not in exist_lfn) and not exists: print('Will create %s' % lpn) add_did(scope=child_scope, name=lpn, type=DIDType.CONTAINER, account=InternalAccount(account), statuses=None, meta=None, rules=None, lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(lpn) parent_name = lpns[lpns.index(lpn) + 1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope) attachments.append({ 'scope': parent_scope, 'name': parent_name, 'dids': [{ 'scope': child_scope, 'name': lpn }] }) # Finally attach everything attach_dids_to_dids(attachments, account=InternalAccount(account), ignore_duplicate=True, session=session)
def add_files(lfns, account, ignore_availability, vo='def', session=None): """ Bulk add files : - Create the file and replica. - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. - Create all the ascendants of the dataset if they do not exist :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} :param issuer: The issuer account. :param ignore_availability: A boolean to ignore blocklisted sites. :param vo: The VO to act on :param session: The session used """ rule_extension_list = [] attachments = [] # The list of scopes is necessary for the extract_scope filter_ = {'scope': InternalScope(scope='*', vo=vo)} scopes = list_scopes(filter_=filter_, session=session) scopes = [scope.external for scope in scopes] exist_lfn = [] try: lifetime_dict = config_get(section='dirac', option='lifetime', session=session) lifetime_dict = loads(lifetime_dict) except ConfigNotFound: lifetime_dict = {} except JSONDecodeError as err: raise InvalidType('Problem parsing lifetime option in dirac section : %s' % str(err)) except Exception as err: raise RucioException(str(err)) for lfn in lfns: # First check if the file exists filename = lfn['lfn'] lfn_scope, _ = extract_scope(filename, scopes) lfn_scope = InternalScope(lfn_scope, vo=vo) exists, did_type = _exists(lfn_scope, filename) if exists: continue # Get all the ascendants of the file lfn_split = filename.split('/') lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))] lpns.reverse() print(lpns) # The parent must be a dataset. Register it as well as the rule dsn_name = lpns[0] dsn_scope, _ = extract_scope(dsn_name, scopes) dsn_scope = InternalScope(dsn_scope, vo=vo) # Compute lifetime lifetime = None if dsn_scope in lifetime_dict: lifetime = lifetime_dict[dsn_scope] else: for pattern in lifetime_dict: if re.match(pattern, dsn_scope): lifetime = lifetime_dict[pattern] break exists, did_type = _exists(dsn_scope, dsn_name) if exists and did_type == DIDType.CONTAINER: raise UnsupportedOperation('Cannot create %s as dataset' % dsn_name) if (dsn_name not in exist_lfn) and not exists: print('Will create %s' % dsn_name) # to maintain a compatibility between master and LTS-1.26 branches remove keywords for first 3 arguments add_did(dsn_scope, dsn_name, DIDType.DATASET, account=InternalAccount(account, vo=vo), statuses=None, meta=None, rules=[{'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account, vo=vo), 'lifetime': None, 'grouping': 'NONE'}], lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(dsn_name) parent_name = lpns[1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope, vo=vo) attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': dsn_scope, 'name': dsn_name}]}) rule_extension_list.append((dsn_scope, dsn_name)) if lifetime and (dsn_scope, dsn_name) not in rule_extension_list: # Reset the lifetime of the rule to the configured value rule = [rul for rul in list_rules({'scope': dsn_scope, 'name': dsn_name, 'account': InternalAccount(account, vo=vo)}, session=session) if rul['rse_expression'] == 'ANY=true'] if rule: update_rule(rule[0]['id'], options={'lifetime': lifetime}, session=session) rule_extension_list.append((dsn_scope, dsn_name)) # Register the file rse_id = lfn.get('rse_id', None) if not rse_id: raise InvalidType('Missing rse_id') bytes_ = lfn.get('bytes', None) guid = lfn.get('guid', None) adler32 = lfn.get('adler32', None) pfn = lfn.get('pfn', None) files = {'scope': lfn_scope, 'name': filename, 'bytes': bytes_, 'adler32': adler32} if pfn: files['pfn'] = str(pfn) if guid: files['meta'] = {'guid': guid} add_replicas(rse_id=rse_id, files=[files], dataset_meta=None, account=InternalAccount(account, vo=vo), ignore_availability=ignore_availability, session=session) add_rule(dids=[{'scope': lfn_scope, 'name': filename}], account=InternalAccount(account, vo=vo), copies=1, rse_expression=lfn['rse'], grouping=None, weight=None, lifetime=86400, locked=None, subscription_id=None, session=session) attachments.append({'scope': dsn_scope, 'name': dsn_name, 'dids': [{'scope': lfn_scope, 'name': filename}]}) # Now loop over the ascendants of the dataset and created them for lpn in lpns[1:]: child_scope, _ = extract_scope(lpn, scopes) child_scope = InternalScope(child_scope, vo=vo) exists, did_type = _exists(child_scope, lpn) if exists and did_type == DIDType.DATASET: raise UnsupportedOperation('Cannot create %s as container' % lpn) if (lpn not in exist_lfn) and not exists: print('Will create %s' % lpn) add_did(child_scope, lpn, DIDType.CONTAINER, account=InternalAccount(account, vo=vo), statuses=None, meta=None, rules=None, lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(lpn) parent_name = lpns[lpns.index(lpn) + 1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope, vo=vo) attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': child_scope, 'name': lpn}]}) # Finally attach everything attach_dids_to_dids(attachments, account=InternalAccount(account, vo=vo), ignore_duplicate=True, session=session)