class LinksMatrix(object): """ CMS RSE distances according to a set of rules """ def __init__(self, account, auth_type=None, exclude=DEFAULT_EXCLUDE_LINKS, distance=None, phedex_links=False, rselist=None, instance=DEFAULT_PHEDEX_INST, datasvc=DEFAULT_DATASVC_URL): if distance is None: distance = DEFAULT_DISTANCE_RULES self.pcli = PhEDEx(instance=instance, datasvc=datasvc) self.rcli = Client(account=account, auth_type=auth_type) self._get_rselist(rselist) self._get_matrix(distance, phedex_links, exclude) def _get_rselist(self, rselist=None): self.rselist = [] if rselist is None: rselist = [rse['rse'] for rse in self.rcli.list_rses()] for rse in rselist: attrs = self.rcli.list_rse_attributes(rse=rse) try: self.rselist.append({ 'rse': rse, 'pnn': attrs['pnn'], 'type': attrs['cms_type'], 'country': attrs['country'], 'region': attrs.get('region', None) }) except KeyError: logging.warning('No expected attributes for RSE %s. Skipping', rse) def _get_matrix(self, distance, phedex_links, exclude): if phedex_links: matrix = self.pcli.links() else: matrix = {} self.links = {} for src in self.rselist: for dest in self.rselist: src_rse = src['rse'] dest_rse = dest['rse'] src_pnn = src['pnn'] dest_pnn = dest['pnn'] link = -1 # Within site or in defined region, don't consult PhEDEx if dest_pnn == src_pnn: link = distance['site'] elif src['region'] and dest['region'] and src[ 'region'] == dest['region']: if src['country'] == dest['country']: link = distance['region&country'] else: link = distance['region'] elif src_pnn in matrix and dest_pnn in matrix[src_pnn]: # If no information, use PhEDEx info if it exists link = distance['site'] - matrix[src_pnn][dest_pnn] else: if src['country'] == dest['country']: link = distance['country'] else: link = distance['other'] if src_rse not in self.links: self.links[src_rse] = {} self.links[src_rse][dest_rse] = link self._filter_matrix(exclude) def _filter_matrix(self, exclude): for src in self.rselist: for dest in self.rselist: if src['rse'] == dest['rse']: continue for rule in exclude: matched = True for item in rule['src']: if not re.match(rule['src'][item], src[item]): matched = False for item in rule['dest']: if not re.match(rule['dest'][item], dest[item]): matched = False if matched: self.links[src['rse']][dest['rse']] = -1 break def update(self, overwrite=False, disable=True, dry=False, srcselect=r'\S+', dstselect=r'\S+'): """ Updates distances according to what is expected :overwrite: overwrite distance of the links that already exist :disable: set ranking to 0 for the links that should be disabled :dry: dry run """ count = {'checked': [], 'created': [], 'updated': [], 'disabled': []} src_regex = re.compile(srcselect) dst_regex = re.compile(dstselect) for src in self.rselist: srse = src['rse'] logging.info("Setting links from %s to %s other RSEs.", srse, len(self.rselist)) for dest in self.rselist: drse = dest['rse'] if srse == drse or not src_regex.match( srse) or not dst_regex.match(drse): continue count['checked'].append([srse, drse]) # Todo.. doublecheck I'm not reversing things link = self.rcli.get_distance(srse, drse) if srse in self.links and drse in self.links[ srse] and self.links[srse][drse] >= 0: if not link: pars = { 'distance': 1, 'ranking': self.links[srse][drse] } if dry: logging.info( "adding link from %s to %s with %s. Dry Run", srse, drse, str(pars)) else: self.rcli.add_distance(srse, drse, pars) count['created'].append([srse, drse]) elif link and overwrite: if dry: logging.info( "setting distance %s for link from %s to %s. Dry run.", self.links[srse][drse], srse, drse) else: self.rcli.update_distance( srse, drse, { 'ranking': self.links[srse][drse], 'distance': 1 }) count['updated'].append([srse, drse]) elif link and disable: if dry: logging.info("disabling link from %s to %s. Dry run", srse, drse) else: self.rcli.update_distance(srse, drse, { 'ranking': None, 'distance': None, }) count['disabled'].append([srse, drse]) return count
"write": 1, "third_party_copy": 1, "delete": 1 } }, "prefix": "/tmp/rucio_rse/" }, ) for key, value in attr.items(): client.add_rse_attribute(rse, key, value) for rse in prod_rses: for rse2 in prod_rses: if rse2 == rse: continue client.add_distance(rse, rse2, {"distance": 1, "ranking": 1}) client.add_scope(account="root", scope="cms") client.add_account("transfer_ops", "SERVICE", "*****@*****.**") client.add_identity( account="transfer_ops", identity="ddmlab", authtype="USERPASS", email="*****@*****.**", ) # why is this still a workaround? from rucio.core.account import add_account_attribute from rucio.common.types import InternalAccount add_account_attribute(InternalAccount("transfer_ops"), "admin", True)