def transfer(args): src_client = Client(**INSTANCES[args.src]) dst_client = Client(**INSTANCES[args.dst]) dids = (did.split(':') for did in args.DIDs) dids = [{'scope': d[0], 'name': d[1]} for d in dids] replicas = src_client.list_replicas(dids, rse_expression=args.rse) available = [] for replica in replicas: if replica['states'][args.rse] == 'AVAILABLE': available.append({ 'scope': replica['scope'], 'name': replica['name'] }) if args.verbose: print('Available replica:', replica['name']) elif args.verbose: print('Unavailable replica:', replica) print("Total replicas: %d" % len(available)) if not args.dry: res = dst_client.add_replicas(args.rse, available) if res: print("Replicas added successfully")
def sync_roles_to_rses(): result = requests.get(CRIC_USERS_API, verify=False) # Pods don't like the CRIC certificate all_cric_users = json.loads(result.text) site_managers = defaultdict(set) for user in all_cric_users: roles = user['ROLES'] username = user['LOGIN'] if 'data-manager' in roles: for thing in roles['data-manager']: if thing.startswith('site:'): site = (thing.replace('site:', '', 1)).replace('-', '_') site_managers[site].add(username) client = Client() all_rses = client.list_rses() for rse in all_rses: rse_name = rse['rse'] if rse_name.lower() in site_managers: print("Setting manager for %s" % rse_name) set_rse_manager(client, rse_name, site_managers) else: set_approvers = False for suffix in TO_STRIP: test_name = rse_name.replace(suffix, '', 1) if test_name.lower() in site_managers: print("Setting alternate for %s" % rse_name) set_rse_manager(client, rse_name, site_managers, test_name) set_approvers = True break if not set_approvers: print("No site manager found for %s" % rse_name)
def __init__(self, _client=None, logger=None, tracing=True): """ Initialises the basic settings for an UploadClient object :param _client: - Optional: rucio.client.client.Client object. If None, a new object will be created. :param logger: - Optional: logging.Logger object. If None, default logger will be used. """ if not logger: self.logger = logging.log else: self.logger = logger.log self.client = _client if _client else Client() self.client_location = detect_client_location() # if token should be used, use only JWT tokens self.auth_token = self.client.auth_token if len( self.client.auth_token.split(".")) == 3 else None self.tracing = tracing if not self.tracing: logger(logging.DEBUG, 'Tracing is turned off.') self.default_file_scope = 'user.' + self.client.account self.rses = {} self.rse_expressions = {} self.trace = {} self.trace['hostname'] = socket.getfqdn() self.trace['account'] = self.client.account if self.client.vo != 'def': self.trace['vo'] = self.client.vo self.trace['eventType'] = 'upload' self.trace['eventVersion'] = version.RUCIO_VERSION[0]
def dataset_replica_update(dataset, pnn, rse, pcli, account, dry): """ Just wrapping the update method. """ try: rcli = Client(account=account) except CannotAuthenticate: logging.warning("cannot authenticate with account %s, skipping pnn %s", account, pnn) return None logging.my_fmt(label='update:rse=%s:rds=%s' % (pnn, dataset)) logging.notice('Starting.') try: ret = _replica_update(dataset, pnn, rse, pcli, rcli, dry) #pylint: disable=broad-except except Exception as exc: logging.error('Exception %s raised: %s', type(exc).__name__, traceback.format_exc().replace('\n', '~~')) return None logging.notice('Finished %s.', ret)
def __init__(self, _client=None, logger=None, tracing=True): """ Initialises the basic settings for an UploadClient object :param _client: - Optional: rucio.client.client.Client object. If None, a new object will be created. :param logger: - logging.Logger object to use for uploads. If None nothing will be logged. """ if not logger: logger = logging.getLogger('%s.null' % __name__) logger.disabled = True self.logger = logger self.client = _client if _client else Client() self.client_location = detect_client_location() # if token should be used, use only JWT tokens self.auth_token = self.client.auth_token if len( self.client.auth_token.split(".")) == 3 else None self.tracing = tracing if not self.tracing: logger.debug('Tracing is turned off.') self.default_file_scope = 'user.' + self.client.account self.rses = {} self.trace = {} self.trace['hostname'] = socket.getfqdn() self.trace['account'] = self.client.account self.trace['eventType'] = 'upload' self.trace['eventVersion'] = version.RUCIO_VERSION[0]
def sync_roles_to_rses(): all_cric_users = json.load( urllib2.urlopen(CRIC_USERS_API, context=SSL_CONTEXT)) site_managers = defaultdict(set) for user in all_cric_users: roles = user['ROLES'] username = user['LOGIN'] if 'data-manager' in roles: for thing in roles['data-manager']: if thing.startswith('site:'): site = (thing.replace('site:', '', 1)).replace('-', '_') site_managers[site].add(username) client = Client() all_rses = client.list_rses() for rse in all_rses: rse_name = rse['rse'] if rse_name.lower() in site_managers: print("Setting manager for %s" % rse_name) set_rse_manager(client, rse_name, site_managers) else: set_approvers = False for suffix in TO_STRIP: test_name = rse_name.replace(suffix, '', 1) if test_name.lower() in site_managers: print("Setting alternate for %s" % rse_name) set_rse_manager(client, rse_name, site_managers, test_name) set_approvers = True break if not set_approvers: print("No site manager found for %s" % rse_name)
def sync_roles_to_group_accounts(): result = requests.get(CRIC_GROUP_API, verify=False) # Pods don't like the CRIC certificate all_cric_groups = json.loads(result.text) client = Client() for group, data in all_cric_groups.items(): if group in role_group_mapping: group_name = role_group_mapping[group]['name'] group_email = role_group_mapping[group]['email'] print('Setting identities for %s' % group_name) try: client.get_account(group_name) except AccountNotFound: print('Adding group account %s with %s' % (group_name, group_email)) client.add_account(group_name, 'GROUP', group_email) group_info = {user['dn']: user['email'] for user in data['users']} current_identities = set(identity['identity'] for identity in client.list_identities(group_name)) target_identities = set(group_info.keys()) add_identities = target_identities - current_identities del_identities = current_identities - target_identities # FIXME: This does not pick up email changes with the same DN for identity in add_identities: print('Adding %s to %s with %s' % (identity, group_name, group_info[identity])) client.add_identity(account=group_name, identity=identity, authtype='X509', email=group_info[identity]) for identity in del_identities: print('Deleting %s from %s' % (identity, group_name)) client.del_identity(account=group_name, identity=identity, authtype='X509')
def test_ping(self): """ PING (CLIENT): Ping Rucio """ creds = {'username': '******', 'password': '******'} client = Client(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, **self.vo) print(client.ping())
def __init__(self, _client=None, user_agent='rucio_clients', logger=None): """ Initialises the basic settings for an DownloadClient object :param _client: Optional: rucio.client.client.Client object. If None, a new object will be created. :param user_agent: user_agent that is using the download client :param logger: logging.Logger object to use for downloads. If None nothing will be logged. """ if not logger: logger = logging.getLogger(__name__).getChild('null') logger.addHandler(logging.NullHandler()) self.logger = logger self.is_human_readable = True self.client = _client if _client else Client() self.user_agent = user_agent account_attributes = [ acc for acc in self.client.list_account_attributes(self.client.account) ] self.is_admin = False for attr in account_attributes[0]: if attr['key'] == 'admin': self.is_admin = attr['value'] is True break if self.is_admin: logger.debug('Admin mode enabled') self.trace_tpl = {} self.trace_tpl['hostname'] = socket.getfqdn() self.trace_tpl['account'] = self.client.account self.trace_tpl['eventType'] = 'download' self.trace_tpl['eventVersion'] = 'api'
def __init__(self, pnn, account, auth_type=None, rsetype=DEFAULT_RSETYPE, suffix=None, dry=False, fts=None, tier=None, lfn2pfn_algorithm=None, country=None, attrs=None, seinfo=None, tfc=None, tfc_exclude=EXCLUDE_TFC, domains=None, space_token=None, add_prefix=None, proto=DEFAULT_PROTOCOL, instance=DEFAULT_PHEDEX_INST, dasgoclient=DEFAULT_DASGOCLIENT, datasvc=DEFAULT_DATASVC_URL): attrs = attrs or [] self.pnn = pnn self.rsetype = rsetype if suffix is None: suffix = DEFAULT_SUFFIXES[rsetype] self.suffix = suffix if pnn.endswith('_MSS'): raise ValueError( 'Please import PhEDEx _Buffer pnns rather than _MSS for tape endpoints' ) elif pnn.endswith('_Buffer'): self.rsename = pnn.replace('_Buffer', '_Tape') + self.suffix self.rucio_rse_type = 'TAPE' else: self.rsename = pnn + self.suffix self.rucio_rse_type = 'DISK' if tfc and os.path.isdir(tfc): self.tfc = tfc + '/' + pnn + '/PhEDEx/storage.xml' else: self.tfc = tfc self.pcli = PhEDEx(instance=instance, dasgoclient=dasgoclient, datasvc=datasvc) self.rcli = Client(account=account, auth_type=auth_type) self.dry = dry self._get_attributes(fts, tier, lfn2pfn_algorithm, country, attrs) self._get_settings() self._get_protocol(seinfo, add_prefix, tfc_exclude, domains, space_token, proto)
def __init__(self, pnn, account, auth_type=None, rsetype=DEFAULT_RSETYPE, suffix=None, dry=False, fts=None, tier=None, lfn2pfn_algorithm=None, country=None, attrs=None, seinfo=None, tfc=None, tfc_exclude=EXCLUDE_TFC, domains=None, space_token=None, add_prefix=None, proto=DEFAULT_PROTOCOL, instance=DEFAULT_PHEDEX_INST, dasgoclient=DEFAULT_DASGOCLIENT, datasvc=DEFAULT_DATASVC_URL): attrs = attrs or [] self.pnn = pnn self.rsetype = rsetype if suffix is None: suffix = DEFAULT_SUFFIXES[rsetype] self.suffix = suffix self.rsename = pnn + self.suffix if tfc and os.path.isdir(tfc): self.tfc = tfc + '/' + pnn + '/PhEDEx/storage.xml' else: self.tfc = tfc self.pcli = PhEDEx(instance=instance, dasgoclient=dasgoclient, datasvc=datasvc) self.rcli = Client(account=account, auth_type=auth_type) self.dry = dry self._get_attributes(fts, tier, lfn2pfn_algorithm, country, attrs) self._get_settings() self._get_protocol(seinfo, add_prefix, tfc_exclude, domains, space_token, proto)
def __init__(self, json, dry=False, cms_type='real', deterministic=True): self.json = json self.dry = dry self.cms_type = cms_type self.rcli = Client() self.protocols = [] self.attrs = {} self.settings = {} self.settings['deterministic'] = deterministic self.rucio_rse_type = json['type'].upper() # If we are building a _Test or _Temp instance add the special prefix if cms_type == "test": self.rse_name = json['rse'] + "_Test" elif cms_type == "temp": self.rse_name = json['rse'] + "_Temp" else: self.rse_name = json['rse'] # pdb.set_trace() self._get_attributes() self.attrs['fts'] = ','.join(json['fts'])
def __init__(self, client=None, logger=None): """ Initialises the basic settings for an DownloadClient object :param client: Optional: rucio.client.client.Client object. If None, a new object will be created. :param logger: Optional: logging.Logger object to use for downloads. If None nothing will be logged. """ if not logger: logger = logging.getLogger('%s.null' % __name__) logger.disabled = True self.logger = logger self.is_human_readable = True self.client = client if client else Client() self.client_location = detect_client_location() account_attributes = [acc for acc in self.client.list_account_attributes(self.client.account)] self.is_admin = False for attr in account_attributes[0]: if attr['key'] == 'admin': self.is_admin = attr['value'] is True break if self.is_admin: logger.debug('Admin mode enabled') self.trace_tpl = {} self.trace_tpl['hostname'] = self.client_location['fqdn'] self.trace_tpl['localSite'] = self.client_location['site'] self.trace_tpl['account'] = self.client.account self.trace_tpl['eventType'] = 'download' self.trace_tpl['eventVersion'] = 'api_' + version.RUCIO_VERSION[0]
def _launch_workers(pnns, datasets, pool, options, pcli): procs = [] rcli = Client() for pnn in pnns: account = options.account or SYNC_ACCOUNT_FMT % pnn.lower() # try: # rcli = Client(account=account) # except CannotAuthenticate: # logging.warning("cannot authenticate with account %s, skipping pnn %s", # account, pnn) # continue rse = list(rcli.list_rses('pnn=%s&cms_type=real' % pnn)) if not rse: logging.warning("cannot find real rse for pnn %s, skipping", pnn) continue rse = rse[0]['rse'] for dataset in datasets: procs.append( pool.apply_async( dataset_replica_update, (dataset, pnn, rse, pcli, account, options.dry))) return procs
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} else: self.vo = {} logger = logging.getLogger('dlul_client') logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) self.client = Client() self.upload_client = UploadClient(_client=self.client, logger=logger) self.download_client = DownloadClient(client=self.client, logger=logger) self.file_path = file_generator() self.scope = 'mock' self.name = os.path.basename(self.file_path) self.rse = 'MOCK4' self.guid = generate_uuid() item = {'path': self.file_path, 'rse': self.rse, 'did_scope': self.scope, 'did_name': self.name, 'guid': self.guid} assert self.upload_client.upload([item]) == 0
def __init__(self, _client=None, user_agent='rucio_clients', logger=None): """ Initialises the basic settings for an UploadClient object :param _client: - Optional: rucio.client.client.Client object. If None, a new object will be created. :param user_agent: - user_agent that is using the upload client :param logger: - logging.Logger object to use for uploads. If None nothing will be logged. """ if not logger: logger = logging.getLogger(__name__).getChild('null') logger.addHandler(logging.NullHandler()) self.logger = logger self.client = _client if _client else Client() self.account = self.client.account self.user_agent = user_agent self.default_file_scope = 'user.' + self.client.account self.rses = {} self.trace = {} self.trace['hostname'] = socket.getfqdn() self.trace['account'] = self.account self.trace['eventType'] = 'upload' self.trace['eventVersion'] = version.RUCIO_VERSION[0]
def setup(self): logger = logging.getLogger('dlul_client') logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) self.client = Client() self.upload_client = UploadClient(_client=self.client, logger=logger) self.download_client = DownloadClient(client=self.client, logger=logger)
def test_get_vo_from_config(self): """ MULTI VO (CLIENT): Get vo from config file when starting clients """ # Start clients with vo explicitly set to None replica_client = ReplicaClient(vo=None) client = Client(vo=None) upload_client = UploadClient(_client=client) # Check the vo has been got from the config file assert_equal(replica_client.vo, self.vo['vo']) assert_equal(upload_client.client.vo, self.vo['vo'])
def get_rucio_client(self): try: client = Client() except CannotAuthenticate as error: self.logger.error(error) self.logger.error(traceback.format_exc()) raise exceptions.AgentPluginError( '%s: %s' % (str(error), traceback.format_exc())) return client
def pre_cache(self, scope, name): """ Pre cache the dataset to this edge. The edge should define a path or a storage for pre caching. """ item = { 'did': '%s:%s' % (scope, name), 'base_dir': self.cache_path, 'no_subdir': self.no_subdir, 'transfer_timeout': self.transfer_timeout } if self.rse: item['rse'] = self.rse client = Client() all_files = client.list_files(scope, name) download_client = DownloadClient(client=client) downloaded_files = download_client.download_dids( [item], num_threads=self.num_threads) self.logger.info('Downloaded files: %s' % downloaded_files) ret_files = [] for file in all_files: downloaded_file = None for d_file in downloaded_files: if d_file['scope'] == file['scope'] and d_file['name'] == file[ 'name']: downloaded_file = d_file break ret_file = { 'scope': file['scope'], 'name': file['name'], 'min_id': 1, 'max_id': file['events'], 'status': ContentStatus.AVAILABLE if downloaded_file and downloaded_file['clientState'] == 'ALREADY_DONE' else ContentStatus.NEW, 'size': file['bytes'], 'md5': downloaded_file['md5'] if downloaded_file else None, 'adler32': downloaded_file['adler32'] if downloaded_file else None, 'pfn': downloaded_file['dest_file_path'] if downloaded_file else None } ret_files.append(ret_file) return ret_files
def __init__(self, account, auth_type, creds=None, scope='cms', dry_run=False, das_go_path=DEFAULT_DASGOCLIENT, check=False): self.account = account self.auth_type = auth_type self.creds = creds self.scope = scope self.dry_run = dry_run self.dasgoclient = das_go_path self.check = check self.cli = Client(account=self.account, auth_type=self.auth_type, creds=self.creds)
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} else: self.vo = {} logger = logging.getLogger('dlul_client') logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) self.client = Client() self.did_client = DIDClient() self.upload_client = UploadClient(_client=self.client, logger=logger) self.download_client = DownloadClient(client=self.client, logger=logger)
def _get_rcli(self, rcli): if rcli is None: rcli = {} if isinstance(rcli, dict): if 'account' not in rcli: rcli['account'] = SYNC_ACCOUNT_FMT % self.pnn.lower() self.rcli = Client(**rcli) elif isinstance(rcli, Client): # pylint: disable=redefined-variable-type self.rcli = rcli else: raise Exception("wrong type for rcli parameter %s" % type(rcli))
def __init__(self): """ SonarTest class constructor. """ self.client = Client() self.endpoint_names = [] self.rule_times = {} self.rule_dict = {} self.traffic_weights = {} self.traffic_data = {} self.get_traffic_data() self.get_rule_data() self.update_available_site_list() self.add_replication_delay() self.traffic_weights = self.calculate_weights()
def run_distribution(): """ Every x hours tries to distribute the datasets to RSE's that are missing them. """ client = Client() counter = 0 dataset_dir = config_get('sonar', 'dataset_dir') dataset_prefix = config_get('sonar', 'dataset_prefix') scope = config_get('sonar', 'scope') num_files = 10 while not GRACEFUL_STOP.is_set(): if counter % 12 == 0: distribute_files(client, data_dir=dataset_dir, dataset_prefix=dataset_prefix, scope=scope, num_files=num_files) time.sleep(3600) counter += 1
def _ping(rcli=None): """ pings the Rucio server """ if rcli is None: rcli = Client() try: rcli.ping() except RucioException as exc: logging.verbose('Ping Exception %s, %s', type(exc).__name__, traceback.format_exc().replace('\n', '~~')) return False return True
def __init__(self, account, auth_type=None, exclude=DEFAULT_EXCLUDE_LINKS, distance=None, phedex_links=False, rselist=None): if distance is None: distance = DEFAULT_DISTANCE_RULES self.pcli = None self.rcli = Client(account=account, auth_type=auth_type) self._get_rselist(rselist) self._get_matrix(distance, phedex_links, exclude)
def __init__(self, _client=None, user_agent='rucio_clients', logger=None): if not logger: logger = logging.getLogger('rucio_uploadclient') logger.addHandler(logging.NullHandler()) self.logger = logger self.client = _client if _client else Client() self.account = self.client.account self.user_agent = user_agent self.default_file_scope = 'user.' + self.client.account self.rses = {} self.trace = {} self.trace['hostname'] = socket.getfqdn() self.trace['account'] = self.account self.trace['eventType'] = 'upload' self.trace['eventVersion'] = version.RUCIO_VERSION[0]
def __init__(self, myscope, orgRse, destRse, account='bruzzese', working_folder=None): self.myscope = myscope self.orgRse = orgRse self.destRse = destRse self.working_folder = working_folder self.gfal = Gfal2Context() self.didc = DIDClient() self.repc = ReplicaClient() self.rulesClient = RuleClient() # Configuration self.account = account # account=account self.client = Client(account=self.account)
def __init__(self, rses=None, rsefilter=None, identity=None): self.rcli = Client(account='root', auth_type=None) self._get_identity(identity) if rses is None: rses = [] for rse in self.rcli.list_rses(): attrs = self.rcli.list_rse_attributes(rse=rse['rse']) if 'cms_type' in attrs and attrs['cms_type'] == 'real': rses.append(rse['rse']) self.accounts = [] for rse in rses: if rsefilter is None or re.match(rsefilter, rse): self.accounts.append(SYNC_ACCOUNT_FMT % rse.lower())