def __init__(self, json, dry=False, cms_type='real', deterministic=True):
self.json = json
self.dry = dry
self.cms_type = cms_type
self.rcli = Client()
self.protocols = []
self.attrs = {}
self.settings = {}
self.settings['deterministic'] = deterministic
self.rucio_rse_type = json['type'].upper()
# If we are building a _Test or _Temp instance add the special prefix
if cms_type == "test":
self.rse_name = json['rse'] + "_Test"
elif cms_type == "temp":
self.rse_name = json['rse'] + "_Temp"
else:
self.rse_name = json['rse']
# pdb.set_trace()
self._get_attributes()
self.attrs['fts'] = ','.join(json['fts'])
def sync_roles_to_rses():
result = requests.get(CRIC_USERS_API,
verify=False) # Pods don't like the CRIC certificate
all_cric_users = json.loads(result.text)
site_managers = defaultdict(set)
for user in all_cric_users:
roles = user['ROLES']
username = user['LOGIN']
if 'data-manager' in roles:
for thing in roles['data-manager']:
if thing.startswith('site:'):
site = (thing.replace('site:', '', 1)).replace('-', '_')
site_managers[site].add(username)
client = Client()
all_rses = client.list_rses()
for rse in all_rses:
rse_name = rse['rse']
if rse_name.lower() in site_managers:
print("Setting manager for %s" % rse_name)
set_rse_manager(client, rse_name, site_managers)
else:
set_approvers = False
for suffix in TO_STRIP:
test_name = rse_name.replace(suffix, '', 1)
if test_name.lower() in site_managers:
print("Setting alternate for %s" % rse_name)
set_rse_manager(client, rse_name, site_managers, test_name)
set_approvers = True
break
if not set_approvers:
print("No site manager found for %s" % rse_name)
def test_ping(self):
""" PING (CLIENT): Ping Rucio """
creds = {'username': '******', 'password': '******'}
client = Client(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, **self.vo)
print(client.ping())
def __init__(self, pnn, account, auth_type=None, rsetype=DEFAULT_RSETYPE, suffix=None,
dry=False, fts=None, tier=None, lfn2pfn_algorithm=None, country=None,
attrs=None, seinfo=None, tfc=None, tfc_exclude=EXCLUDE_TFC, domains=None,
space_token=None, add_prefix=None, proto=DEFAULT_PROTOCOL,
instance=DEFAULT_PHEDEX_INST, dasgoclient=DEFAULT_DASGOCLIENT,
datasvc=DEFAULT_DATASVC_URL):
attrs = attrs or []
self.pnn = pnn
self.rsetype = rsetype
if suffix is None:
suffix = DEFAULT_SUFFIXES[rsetype]
self.suffix = suffix
self.rsename = pnn + self.suffix
if tfc and os.path.isdir(tfc):
self.tfc = tfc + '/' + pnn + '/PhEDEx/storage.xml'
else:
self.tfc = tfc
self.pcli = PhEDEx(instance=instance, dasgoclient=dasgoclient, datasvc=datasvc)
self.rcli = Client(account=account, auth_type=auth_type)
self.dry = dry
self._get_attributes(fts, tier, lfn2pfn_algorithm, country, attrs)
self._get_settings()
self._get_protocol(seinfo, add_prefix, tfc_exclude, domains, space_token, proto)
def test_ping(self):
""" PING (CLIENT): Ping Rucio """
creds = {'username': '******', 'password': '******'}
c = Client(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds)
print c.ping()
def _launch_workers(pnns, datasets, pool, options, pcli):
procs = []
rcli = Client()
for pnn in pnns:
account = options.account or SYNC_ACCOUNT_FMT % pnn.lower()
# try:
# rcli = Client(account=account)
# except CannotAuthenticate:
# logging.warning("cannot authenticate with account %s, skipping pnn %s",
# account, pnn)
# continue
rse = list(rcli.list_rses('pnn=%s&cms_type=real' % pnn))
if not rse:
logging.warning("cannot find real rse for pnn %s, skipping", pnn)
continue
rse = rse[0]['rse']
for dataset in datasets:
procs.append(
pool.apply_async(
dataset_replica_update,
(dataset, pnn, rse, pcli, account, options.dry)))
return procs
def sync_roles_to_rses():
all_cric_users = json.load(
urllib2.urlopen(CRIC_USERS_API, context=SSL_CONTEXT))
site_managers = defaultdict(set)
for user in all_cric_users:
roles = user['ROLES']
username = user['LOGIN']
if 'data-manager' in roles:
for thing in roles['data-manager']:
if thing.startswith('site:'):
site = (thing.replace('site:', '', 1)).replace('-', '_')
site_managers[site].add(username)
client = Client()
all_rses = client.list_rses()
for rse in all_rses:
rse_name = rse['rse']
if rse_name.lower() in site_managers:
print("Setting manager for %s" % rse_name)
set_rse_manager(client, rse_name, site_managers)
else:
set_approvers = False
for suffix in TO_STRIP:
test_name = rse_name.replace(suffix, '', 1)
if test_name.lower() in site_managers:
print("Setting alternate for %s" % rse_name)
set_rse_manager(client, rse_name, site_managers, test_name)
set_approvers = True
break
if not set_approvers:
print("No site manager found for %s" % rse_name)
def __init__(self,
pnn,
account,
auth_type=None,
rsetype=DEFAULT_RSETYPE,
suffix=None,
dry=False,
fts=None,
tier=None,
lfn2pfn_algorithm=None,
country=None,
attrs=None,
seinfo=None,
tfc=None,
tfc_exclude=EXCLUDE_TFC,
domains=None,
space_token=None,
add_prefix=None,
proto=DEFAULT_PROTOCOL,
instance=DEFAULT_PHEDEX_INST,
dasgoclient=DEFAULT_DASGOCLIENT,
datasvc=DEFAULT_DATASVC_URL):
attrs = attrs or []
self.pnn = pnn
self.rsetype = rsetype
if suffix is None:
suffix = DEFAULT_SUFFIXES[rsetype]
self.suffix = suffix
if pnn.endswith('_MSS'):
raise ValueError(
'Please import PhEDEx _Buffer pnns rather than _MSS for tape endpoints'
)
elif pnn.endswith('_Buffer'):
self.rsename = pnn.replace('_Buffer', '_Tape') + self.suffix
self.rucio_rse_type = 'TAPE'
else:
self.rsename = pnn + self.suffix
self.rucio_rse_type = 'DISK'
if tfc and os.path.isdir(tfc):
self.tfc = tfc + '/' + pnn + '/PhEDEx/storage.xml'
else:
self.tfc = tfc
self.pcli = PhEDEx(instance=instance,
dasgoclient=dasgoclient,
datasvc=datasvc)
self.rcli = Client(account=account, auth_type=auth_type)
self.dry = dry
self._get_attributes(fts, tier, lfn2pfn_algorithm, country, attrs)
self._get_settings()
self._get_protocol(seinfo, add_prefix, tfc_exclude, domains,
space_token, proto)
def pre_cache(self, scope, name):
"""
Pre cache the dataset to this edge.
The edge should define a path or a storage for pre caching.
"""
item = {
'did': '%s:%s' % (scope, name),
'base_dir': self.cache_path,
'no_subdir': self.no_subdir,
'transfer_timeout': self.transfer_timeout
}
if self.rse:
item['rse'] = self.rse
client = Client()
all_files = client.list_files(scope, name)
download_client = DownloadClient(client=client)
downloaded_files = download_client.download_dids(
[item], num_threads=self.num_threads)
self.logger.info('Downloaded files: %s' % downloaded_files)
ret_files = []
for file in all_files:
downloaded_file = None
for d_file in downloaded_files:
if d_file['scope'] == file['scope'] and d_file['name'] == file[
'name']:
downloaded_file = d_file
break
ret_file = {
'scope':
file['scope'],
'name':
file['name'],
'min_id':
1,
'max_id':
file['events'],
'status':
ContentStatus.AVAILABLE if downloaded_file
and downloaded_file['clientState'] == 'ALREADY_DONE' else
ContentStatus.NEW,
'size':
file['bytes'],
'md5':
downloaded_file['md5'] if downloaded_file else None,
'adler32':
downloaded_file['adler32'] if downloaded_file else None,
'pfn':
downloaded_file['dest_file_path'] if downloaded_file else None
}
ret_files.append(ret_file)
return ret_files
def __init__(self, account, auth_type, creds=None, scope='cms', dry_run=False,
das_go_path=DEFAULT_DASGOCLIENT, check=False):
self.account = account
self.auth_type = auth_type
self.creds = creds
self.scope = scope
self.dry_run = dry_run
self.dasgoclient = das_go_path
self.check = check
self.cli = Client(account=self.account, auth_type=self.auth_type, creds=self.creds)
def _get_rcli(self, rcli):
if rcli is None:
rcli = {}
if isinstance(rcli, dict):
if 'account' not in rcli:
rcli['account'] = SYNC_ACCOUNT_FMT % self.pnn.lower()
self.rcli = Client(**rcli)
elif isinstance(rcli, Client):
# pylint: disable=redefined-variable-type
self.rcli = rcli
else:
raise Exception("wrong type for rcli parameter %s" % type(rcli))
def __init__(self):
"""
SonarTest class constructor.
"""
self.client = Client()
self.endpoint_names = []
self.rule_times = {}
self.rule_dict = {}
self.traffic_weights = {}
self.traffic_data = {}
self.get_traffic_data()
self.get_rule_data()
self.update_available_site_list()
self.add_replication_delay()
self.traffic_weights = self.calculate_weights()
def __init__(self, _client=None, user_agent='rucio_clients', logger=None):
"""
Initialises the basic settings for an DownloadClient object
:param _client: Optional: rucio.client.client.Client object. If None, a new object will be created.
:param user_agent: user_agent that is using the download client
:param logger: logging.Logger object to use for downloads. If None nothing will be logged.
"""
if not logger:
logger = logging.getLogger(__name__).getChild('null')
logger.addHandler(logging.NullHandler())
self.logger = logger
self.is_human_readable = True
self.client = _client if _client else Client()
self.user_agent = user_agent
account_attributes = [
acc
for acc in self.client.list_account_attributes(self.client.account)
]
self.is_admin = False
for attr in account_attributes[0]:
if attr['key'] == 'admin':
self.is_admin = attr['value'] is True
break
if self.is_admin:
logger.debug('Admin mode enabled')
self.trace_tpl = {}
self.trace_tpl['hostname'] = socket.getfqdn()
self.trace_tpl['account'] = self.client.account
self.trace_tpl['eventType'] = 'download'
self.trace_tpl['eventVersion'] = 'api'
def __init__(self, client=None, logger=None):
"""
Initialises the basic settings for an DownloadClient object
:param client: Optional: rucio.client.client.Client object. If None, a new object will be created.
:param logger: Optional: logging.Logger object to use for downloads. If None nothing will be logged.
"""
if not logger:
logger = logging.getLogger('%s.null' % __name__)
logger.disabled = True
self.logger = logger
self.is_human_readable = True
self.client = client if client else Client()
self.client_location = detect_client_location()
account_attributes = [acc for acc in self.client.list_account_attributes(self.client.account)]
self.is_admin = False
for attr in account_attributes[0]:
if attr['key'] == 'admin':
self.is_admin = attr['value'] is True
break
if self.is_admin:
logger.debug('Admin mode enabled')
self.trace_tpl = {}
self.trace_tpl['hostname'] = self.client_location['fqdn']
self.trace_tpl['localSite'] = self.client_location['site']
self.trace_tpl['account'] = self.client.account
self.trace_tpl['eventType'] = 'download'
self.trace_tpl['eventVersion'] = 'api_' + version.RUCIO_VERSION[0]
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
logger = logging.getLogger('dlul_client')
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.DEBUG)
self.client = Client()
self.upload_client = UploadClient(_client=self.client, logger=logger)
self.download_client = DownloadClient(client=self.client, logger=logger)
self.file_path = file_generator()
self.scope = 'mock'
self.name = os.path.basename(self.file_path)
self.rse = 'MOCK4'
self.guid = generate_uuid()
item = {'path': self.file_path,
'rse': self.rse,
'did_scope': self.scope,
'did_name': self.name,
'guid': self.guid}
assert self.upload_client.upload([item]) == 0
def __init__(self, _client=None, user_agent='rucio_clients', logger=None):
"""
Initialises the basic settings for an UploadClient object
:param _client: - Optional: rucio.client.client.Client object. If None, a new object will be created.
:param user_agent: - user_agent that is using the upload client
:param logger: - logging.Logger object to use for uploads. If None nothing will be logged.
"""
if not logger:
logger = logging.getLogger(__name__).getChild('null')
logger.addHandler(logging.NullHandler())
self.logger = logger
self.client = _client if _client else Client()
self.account = self.client.account
self.user_agent = user_agent
self.default_file_scope = 'user.' + self.client.account
self.rses = {}
self.trace = {}
self.trace['hostname'] = socket.getfqdn()
self.trace['account'] = self.account
self.trace['eventType'] = 'upload'
self.trace['eventVersion'] = version.RUCIO_VERSION[0]
def dataset_replica_update(dataset, pnn, rse, pcli, account, dry):
"""
Just wrapping the update method.
"""
try:
rcli = Client(account=account)
except CannotAuthenticate:
logging.warning("cannot authenticate with account %s, skipping pnn %s",
account, pnn)
return None
logging.my_fmt(label='update:rse=%s:rds=%s' % (pnn, dataset))
logging.notice('Starting.')
try:
ret = _replica_update(dataset, pnn, rse, pcli, rcli, dry)
#pylint: disable=broad-except
except Exception as exc:
logging.error('Exception %s raised: %s',
type(exc).__name__,
traceback.format_exc().replace('\n', '~~'))
return None
logging.notice('Finished %s.', ret)
def __init__(self, _client=None, logger=None, tracing=True):
"""
Initialises the basic settings for an UploadClient object
:param _client: - Optional: rucio.client.client.Client object. If None, a new object will be created.
:param logger: - logging.Logger object to use for uploads. If None nothing will be logged.
"""
if not logger:
logger = logging.getLogger('%s.null' % __name__)
logger.disabled = True
self.logger = logger
self.client = _client if _client else Client()
self.client_location = detect_client_location()
# if token should be used, use only JWT tokens
self.auth_token = self.client.auth_token if len(
self.client.auth_token.split(".")) == 3 else None
self.tracing = tracing
if not self.tracing:
logger.debug('Tracing is turned off.')
self.default_file_scope = 'user.' + self.client.account
self.rses = {}
self.trace = {}
self.trace['hostname'] = socket.getfqdn()
self.trace['account'] = self.client.account
self.trace['eventType'] = 'upload'
self.trace['eventVersion'] = version.RUCIO_VERSION[0]
def __init__(self, _client=None, logger=None, tracing=True):
"""
Initialises the basic settings for an UploadClient object
:param _client: - Optional: rucio.client.client.Client object. If None, a new object will be created.
:param logger: - Optional: logging.Logger object. If None, default logger will be used.
"""
if not logger:
self.logger = logging.log
else:
self.logger = logger.log
self.client = _client if _client else Client()
self.client_location = detect_client_location()
# if token should be used, use only JWT tokens
self.auth_token = self.client.auth_token if len(
self.client.auth_token.split(".")) == 3 else None
self.tracing = tracing
if not self.tracing:
logger(logging.DEBUG, 'Tracing is turned off.')
self.default_file_scope = 'user.' + self.client.account
self.rses = {}
self.rse_expressions = {}
self.trace = {}
self.trace['hostname'] = socket.getfqdn()
self.trace['account'] = self.client.account
if self.client.vo != 'def':
self.trace['vo'] = self.client.vo
self.trace['eventType'] = 'upload'
self.trace['eventVersion'] = version.RUCIO_VERSION[0]
def setup(self):
logger = logging.getLogger('dlul_client')
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.DEBUG)
self.client = Client()
self.upload_client = UploadClient(_client=self.client, logger=logger)
self.download_client = DownloadClient(client=self.client,
logger=logger)
def __init__(self,
account,
auth_type=None,
exclude=DEFAULT_EXCLUDE_LINKS,
distance=None,
phedex_links=False,
rselist=None):
if distance is None:
distance = DEFAULT_DISTANCE_RULES
self.pcli = None
self.rcli = Client(account=account, auth_type=auth_type)
self._get_rselist(rselist)
self._get_matrix(distance, phedex_links, exclude)
def _ping(rcli=None):
"""
pings the Rucio server
"""
if rcli is None:
rcli = Client()
try:
rcli.ping()
except RucioException as exc:
logging.verbose('Ping Exception %s, %s',
type(exc).__name__,
traceback.format_exc().replace('\n', '~~'))
return False
return True
def find_dataset(self, scope, name):
"""
Find the dataset/container with scope:name.
"""
d = Client()
info = d.get_did(scope, name)
ret = {
'collection_type': info['type'],
'size': info['bytes'],
'total_files': info['length']
}
if info['type'] in ['FILE']:
ret['status'] = 'AVAILABLE'
else:
ret['status'] = 'OPEN' if info['open'] else 'CLOSED'
return ret
def __init__(self, myscope, orgRse, destRse, account='bruzzese', working_folder=None):
self.myscope = myscope
self.orgRse = orgRse
self.destRse = destRse
self.working_folder = working_folder
self.gfal = Gfal2Context()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.rulesClient = RuleClient()
# Configuration
self.account = account
# account=account
self.client = Client(account=self.account)
def get_rucio_client(self):
try:
client = Client()
except CannotAuthenticate as error:
self.logger.error(error)
self.logger.error(traceback.format_exc())
raise exceptions.AgentPluginError(
'%s: %s' % (str(error), traceback.format_exc()))
return client
def test_get_vo_from_config(self):
""" MULTI VO (CLIENT): Get vo from config file when starting clients """
# Start clients with vo explicitly set to None
replica_client = ReplicaClient(vo=None)
client = Client(vo=None)
upload_client = UploadClient(_client=client)
# Check the vo has been got from the config file
assert_equal(replica_client.vo, self.vo['vo'])
assert_equal(upload_client.client.vo, self.vo['vo'])
def __init__(self, rses=None, rsefilter=None, identity=None):
self.rcli = Client(account='root', auth_type=None)
self._get_identity(identity)
if rses is None:
rses = []
for rse in self.rcli.list_rses():
attrs = self.rcli.list_rse_attributes(rse=rse['rse'])
if 'cms_type' in attrs and attrs['cms_type'] == 'real':
rses.append(rse['rse'])
self.accounts = []
for rse in rses:
if rsefilter is None or re.match(rsefilter, rse):
self.accounts.append(SYNC_ACCOUNT_FMT % rse.lower())
def __init__(self,
account,
auth_type=None,
exclude=DEFAULT_EXCLUDE_LINKS,
distance=None,
phedex_links=False,
rselist=None,
instance=DEFAULT_PHEDEX_INST,
datasvc=DEFAULT_DATASVC_URL):
if distance is None:
distance = DEFAULT_DISTANCE_RULES
self.pcli = PhEDEx(instance=instance, datasvc=datasvc)
self.rcli = Client(account=account, auth_type=auth_type)
self._get_rselist(rselist)
self._get_matrix(distance, phedex_links, exclude)
def transfer(args):
src_client = Client(**INSTANCES[args.src])
dst_client = Client(**INSTANCES[args.dst])
dids = (did.split(':') for did in args.DIDs)
dids = [{'scope': d[0], 'name': d[1]} for d in dids]
replicas = src_client.list_replicas(dids, rse_expression=args.rse)
available = []
for replica in replicas:
if replica['states'][args.rse] == 'AVAILABLE':
available.append({
'scope': replica['scope'],
'name': replica['name']
})
if args.verbose:
print('Available replica:', replica['name'])
elif args.verbose:
print('Unavailable replica:', replica)
print("Total replicas: %d" % len(available))
if not args.dry:
res = dst_client.add_replicas(args.rse, available)
if res:
print("Replicas added successfully")
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
logger = logging.getLogger('dlul_client')
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.DEBUG)
self.client = Client()
self.did_client = DIDClient()
self.upload_client = UploadClient(_client=self.client, logger=logger)
self.download_client = DownloadClient(client=self.client, logger=logger)
def ConfigHost(self):
"""Function: ConfigHost
This member function setup the rucioAPI backend.
To ensure full functionality, it needs:
* Client()
* UploadClient()
* DownloadClient()
:raise Exception if Rucio API is not ready (miss-configured)
"""
try:
self._rucio_client = Client()
self._rucio_client_upload = UploadClient(logger=logger)
# self._rucio_client_upload = UploadClient(tracing=False)
# print("Tracing set to False")
self._rucio_client_download = DownloadClient()
self._rucio_ping = self._rucio_client.ping
except:
print("Can not init the Rucio API")
print("-> Check for your Rucio installation")
exit(1)
