class LinksMatrix(object):
"""
CMS RSE distances according to a set of rules
"""
def __init__(self,
account,
auth_type=None,
exclude=DEFAULT_EXCLUDE_LINKS,
distance=None,
phedex_links=False,
rselist=None,
instance=DEFAULT_PHEDEX_INST,
datasvc=DEFAULT_DATASVC_URL):
if distance is None:
distance = DEFAULT_DISTANCE_RULES
self.pcli = PhEDEx(instance=instance, datasvc=datasvc)
self.rcli = Client(account=account, auth_type=auth_type)
self._get_rselist(rselist)
self._get_matrix(distance, phedex_links, exclude)
def _get_rselist(self, rselist=None):
self.rselist = []
if rselist is None:
rselist = [rse['rse'] for rse in self.rcli.list_rses()]
for rse in rselist:
attrs = self.rcli.list_rse_attributes(rse=rse)
try:
self.rselist.append({
'rse': rse,
'pnn': attrs['pnn'],
'type': attrs['cms_type'],
'country': attrs['country'],
'region': attrs.get('region', None)
})
except KeyError:
logging.warning('No expected attributes for RSE %s. Skipping',
rse)
def _get_matrix(self, distance, phedex_links, exclude):
if phedex_links:
matrix = self.pcli.links()
else:
matrix = {}
self.links = {}
for src in self.rselist:
for dest in self.rselist:
src_rse = src['rse']
dest_rse = dest['rse']
src_pnn = src['pnn']
dest_pnn = dest['pnn']
link = -1
# Within site or in defined region, don't consult PhEDEx
if dest_pnn == src_pnn:
link = distance['site']
elif src['region'] and dest['region'] and src[
'region'] == dest['region']:
if src['country'] == dest['country']:
link = distance['region&country']
else:
link = distance['region']
elif src_pnn in matrix and dest_pnn in matrix[src_pnn]:
# If no information, use PhEDEx info if it exists
link = distance['site'] - matrix[src_pnn][dest_pnn]
else:
if src['country'] == dest['country']:
link = distance['country']
else:
link = distance['other']
if src_rse not in self.links:
self.links[src_rse] = {}
self.links[src_rse][dest_rse] = link
self._filter_matrix(exclude)
def _filter_matrix(self, exclude):
for src in self.rselist:
for dest in self.rselist:
if src['rse'] == dest['rse']:
continue
for rule in exclude:
matched = True
for item in rule['src']:
if not re.match(rule['src'][item], src[item]):
matched = False
for item in rule['dest']:
if not re.match(rule['dest'][item], dest[item]):
matched = False
if matched:
self.links[src['rse']][dest['rse']] = -1
break
def update(self,
overwrite=False,
disable=True,
dry=False,
srcselect=r'\S+',
dstselect=r'\S+'):
"""
Updates distances according to what is expected
:overwrite: overwrite distance of the links that already exist
:disable: set ranking to 0 for the links that should be disabled
:dry: dry run
"""
count = {'checked': [], 'created': [], 'updated': [], 'disabled': []}
src_regex = re.compile(srcselect)
dst_regex = re.compile(dstselect)
for src in self.rselist:
srse = src['rse']
logging.info("Setting links from %s to %s other RSEs.", srse,
len(self.rselist))
for dest in self.rselist:
drse = dest['rse']
if srse == drse or not src_regex.match(
srse) or not dst_regex.match(drse):
continue
count['checked'].append([srse, drse])
# Todo.. doublecheck I'm not reversing things
link = self.rcli.get_distance(srse, drse)
if srse in self.links and drse in self.links[
srse] and self.links[srse][drse] >= 0:
if not link:
pars = {
'distance': 1,
'ranking': self.links[srse][drse]
}
if dry:
logging.info(
"adding link from %s to %s with %s. Dry Run",
srse, drse, str(pars))
else:
self.rcli.add_distance(srse, drse, pars)
count['created'].append([srse, drse])
elif link and overwrite:
if dry:
logging.info(
"setting distance %s for link from %s to %s. Dry run.",
self.links[srse][drse], srse, drse)
else:
self.rcli.update_distance(
srse, drse, {
'ranking': self.links[srse][drse],
'distance': 1
})
count['updated'].append([srse, drse])
elif link and disable:
if dry:
logging.info("disabling link from %s to %s. Dry run",
srse, drse)
else:
self.rcli.update_distance(srse, drse, {
'ranking': None,
'distance': None,
})
count['disabled'].append([srse, drse])
return count
"write": 1,
"third_party_copy": 1,
"delete": 1
}
},
"prefix": "/tmp/rucio_rse/"
},
)
for key, value in attr.items():
client.add_rse_attribute(rse, key, value)
for rse in prod_rses:
for rse2 in prod_rses:
if rse2 == rse:
continue
client.add_distance(rse, rse2, {"distance": 1, "ranking": 1})
client.add_scope(account="root", scope="cms")
client.add_account("transfer_ops", "SERVICE", "*****@*****.**")
client.add_identity(
account="transfer_ops",
identity="ddmlab",
authtype="USERPASS",
email="*****@*****.**",
)
# why is this still a workaround?
from rucio.core.account import add_account_attribute
from rucio.common.types import InternalAccount
add_account_attribute(InternalAccount("transfer_ops"), "admin", True)
