def select_algorithm(algorithm, rule_ids, params): """ Method used in case of chained subscriptions :param algorithm: Algorithm used for the chained rule. Now only associated_site associated_site : Choose an associated endpoint according to the RSE attribute assoiciated_site :param rule_ids: List of parent rules :param params: List of rules parameters to be used by the algorithm """ selected_rses = {} if algorithm == 'associated_site': for rule_id in rule_ids: rule = get_rule(rule_id) logging.debug('In select_algorithm, %s', str(rule)) rse = rule['rse_expression'] vo = rule['account'].vo if rse_exists(rse, vo=vo): rse_id = get_rse_id(rse, vo=vo) rse_attributes = list_rse_attributes(rse_id) associated_sites = rse_attributes.get('associated_sites', None) associated_site_idx = params.get('associated_site_idx', None) if not associated_site_idx: raise SubscriptionWrongParameter('Missing parameter associated_site_idx') if associated_sites: associated_sites = associated_sites.split(',') if associated_site_idx > len(associated_sites) + 1: raise SubscriptionWrongParameter('Parameter associated_site_idx is out of range') associated_site = associated_sites[associated_site_idx - 1] selected_rses[associated_site] = {'source_replica_expression': rse, 'weight': None} else: raise SubscriptionWrongParameter('Algorithm associated_site only works with split_rule') if rule['copies'] != 1: raise SubscriptionWrongParameter('Algorithm associated_site only works with split_rule') return selected_rses
def select_algorithm(algorithm, rule_ids, params): """ Method used in case of chained subscriptions :param algorithm: Algorithm used for the chained rule. Now only associated_site associated_site : Choose an associated endpoint according to the RSE attribute assoiciated_site :param rule_ids: List of parent rules :param params: List of rules parameters to be used by the algorithm """ selected_rses = {} if algorithm == "associated_site": for rule_id in rule_ids: rule = get_rule(rule_id) logging.debug("In select_algorithm, %s", str(rule)) rse = rule["rse_expression"] vo = rule["account"].vo if rse_exists(rse, vo=vo): rse_id = get_rse_id(rse, vo=vo) rse_attributes = list_rse_attributes(rse_id) associated_sites = rse_attributes.get("associated_sites", None) associated_site_idx = params.get("associated_site_idx", None) if not associated_site_idx: raise SubscriptionWrongParameter( "Missing parameter associated_site_idx") if associated_sites: associated_sites = associated_sites.split(",") if associated_site_idx > len(associated_sites) + 1: raise SubscriptionWrongParameter( "Parameter associated_site_idx is out of range") associated_site = associated_sites[associated_site_idx - 1] selected_rses[associated_site] = { "source_replica_expression": rse, "weight": None, } else: raise SubscriptionWrongParameter( "Algorithm associated_site only works with split_rule") if rule["copies"] != 1: raise SubscriptionWrongParameter( "Algorithm associated_site only works with split_rule") return selected_rses
def import_data(data, session=None): """ Import data to add and update records in Rucio. :param data: data to be imported as dictionary. :param session: database session in use. """ # RSEs rses = data.get('rses') if rses: for rse in rses: protocols = rse.get('protocols') if protocols: protocols = protocols.get('protocols') del rse['protocols'] rse_name = rse['rse'] del rse['rse'] if not rse_module.rse_exists(rse_name, session=session): rse_module.add_rse(rse_name, deterministic=rse.get('deterministic'), volatile=rse.get('volatile'), city=rse.get('city'), region_code=rse.get('region_code'), country_name=rse.get('country_name'), staging_area=rse.get('staging_area'), continent=rse.get('continent'), time_zone=rse.get('time_zone'), ISP=rse.get('ISP'), rse_type=rse.get('rse_type'), latitude=rse.get('latitude'), longitude=rse.get('longitude'), ASN=rse.get('ASN'), availability=rse.get('availability'), session=session) else: rse_module.update_rse(rse_name, rse, session=session) # Protocols if protocols: old_protocols = rse_module.get_rse_protocols(rse=rse_name, session=session) for protocol in protocols: scheme = protocol.get('scheme') hostname = protocol.get('hostname') port = protocol.get('port') intersection = [ old_protocol for old_protocol in old_protocols['protocols'] if old_protocol['scheme'] == scheme and old_protocol['hostname'] == hostname and old_protocol['port'] == port ] if intersection: del protocol['scheme'] del protocol['hostname'] del protocol['port'] rse_module.update_protocols(rse=rse_name, scheme=scheme, data=protocol, hostname=hostname, port=port, session=session) else: rse_module.add_protocol(rse=rse_name, parameter=protocol, session=session) # Limits limits = rse.get('limits') if limits: old_limits = rse_module.get_rse_limits(rse=rse_name, session=session) for limit in limits: if limit in old_limits: rse_module.delete_rse_limit(rse=rse_name, name=limit, session=session) rse_module.set_rse_limits(rse=rse_name, name=limit, value=limits[limit], session=session) # Transfer limits transfer_limits = rse.get('transfer_limits') if transfer_limits: for limit in transfer_limits: old_transfer_limits = rse_module.get_rse_transfer_limits( rse=rse_name, activity=limit, session=session) if limit in old_transfer_limits: rse_module.delete_rse_transfer_limits(rse=rse_name, activity=limit, session=session) max_transfers = transfer_limits[limit].items( )[0][1]['max_transfers'] rse_module.set_rse_transfer_limits( rse=rse_name, activity=limit, max_transfers=max_transfers, session=session) # Attributes attributes = rse.get('attributes') if attributes: old_attributes = rse_module.list_rse_attributes( rse=rse_name, session=session) for attr in attributes: if attr in old_attributes: rse_module.del_rse_attribute(rse=rse_name, key=attr, session=session) rse_module.add_rse_attribute(rse=rse_name, key=attr, value=attributes[attr], session=session) # Distances distances = data.get('distances') if distances: for src_rse_name in distances: src = rse_module.get_rse_id(src_rse_name, session=session) for dest_rse_name in distances[src_rse_name]: dest = rse_module.get_rse_id(dest_rse_name, session=session) distance = distances[src_rse_name][dest_rse_name] del distance['src_rse_id'] del distance['dest_rse_id'] old_distance = distance_module.get_distances(src_rse_id=src, dest_rse_id=dest, session=session) if old_distance: distance_module.update_distances(src_rse_id=src, dest_rse_id=dest, parameters=distance, session=session) else: distance_module.add_distance( src_rse_id=src, dest_rse_id=dest, ranking=distance.get('ranking'), agis_distance=distance.get('agis_distance'), geoip_distance=distance.get('geoip_distance'), active=distance.get('active'), submitted=distance.get('submitted'), transfer_speed=distance.get('transfer_speed'), finished=distance.get('finished'), failed=distance.get('failed'), session=session)
def import_rses(rses, rse_sync_method='edit', attr_sync_method='edit', protocol_sync_method='edit', vo='def', session=None): new_rses = [] for rse_name in rses: rse = rses[rse_name] if isinstance(rse.get('rse_type'), string_types): rse['rse_type'] = RSEType(rse['rse_type']) if rse_module.rse_exists(rse_name, vo=vo, include_deleted=False, session=session): # RSE exists and is active rse_id = rse_module.get_rse_id(rse=rse_name, vo=vo, session=session) selected_rse_properties = { key: rse[key] for key in rse if key in rse_module.MUTABLE_RSE_PROPERTIES } rse_module.update_rse(rse_id=rse_id, parameters=selected_rse_properties, session=session) elif rse_module.rse_exists(rse_name, vo=vo, include_deleted=True, session=session): # RSE exists but in deleted state # Should only modify the RSE if importer is configured for edit or hard sync if rse_sync_method in ['edit', 'hard']: rse_id = rse_module.get_rse_id(rse=rse_name, vo=vo, include_deleted=True, session=session) rse_module.restore_rse(rse_id, session=session) selected_rse_properties = { key: rse[key] for key in rse if key in rse_module.MUTABLE_RSE_PROPERTIES } rse_module.update_rse(rse_id=rse_id, parameters=selected_rse_properties, session=session) else: # Config is in RSE append only mode, should not modify the disabled RSE continue else: rse_id = rse_module.add_rse(rse=rse_name, vo=vo, deterministic=rse.get('deterministic'), volatile=rse.get('volatile'), city=rse.get('city'), region_code=rse.get('region_code'), country_name=rse.get('country_name'), staging_area=rse.get('staging_area'), continent=rse.get('continent'), time_zone=rse.get('time_zone'), ISP=rse.get('ISP'), rse_type=rse.get('rse_type'), latitude=rse.get('latitude'), longitude=rse.get('longitude'), ASN=rse.get('ASN'), availability=rse.get('availability'), session=session) new_rses.append(rse_id) # Protocols new_protocols = rse.get('protocols') if new_protocols: # update existing, add missing and remove left over protocols old_protocols = [{ 'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port'] } for protocol in rse_module.get_rse_protocols( rse_id=rse_id, session=session)['protocols']] missing_protocols = [ new_protocol for new_protocol in new_protocols if { 'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port'] } not in old_protocols ] outdated_protocols = [ new_protocol for new_protocol in new_protocols if { 'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port'] } in old_protocols ] new_protocols = [{ 'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port'] } for protocol in new_protocols] to_be_removed_protocols = [ old_protocol for old_protocol in old_protocols if old_protocol not in new_protocols ] if protocol_sync_method == 'append': outdated_protocols = [] for protocol in outdated_protocols: scheme = protocol['scheme'] port = protocol['port'] hostname = protocol['hostname'] del protocol['scheme'] del protocol['hostname'] del protocol['port'] rse_module.update_protocols(rse_id=rse_id, scheme=scheme, data=protocol, hostname=hostname, port=port, session=session) for protocol in missing_protocols: rse_module.add_protocol(rse_id=rse_id, parameter=protocol, session=session) if protocol_sync_method == 'hard': for protocol in to_be_removed_protocols: scheme = protocol['scheme'] port = protocol['port'] hostname = protocol['hostname'] rse_module.del_protocols(rse_id=rse_id, scheme=scheme, port=port, hostname=hostname, session=session) # Limits old_limits = rse_module.get_rse_limits(rse_id=rse_id, session=session) for limit_name in ['MaxBeingDeletedFiles', 'MinFreeSpace']: limit = rse.get(limit_name) if limit: if limit_name in old_limits: rse_module.delete_rse_limits(rse_id=rse_id, name=limit_name, session=session) rse_module.set_rse_limits(rse_id=rse_id, name=limit_name, value=limit, session=session) # Attributes attributes = rse.get('attributes', {}) attributes['lfn2pfn_algorithm'] = rse.get('lfn2pfn_algorithm') attributes['verify_checksum'] = rse.get('verify_checksum') old_attributes = rse_module.list_rse_attributes(rse_id=rse_id, session=session) missing_attributes = [ attribute for attribute in old_attributes if attribute not in attributes ] for attr in attributes: value = attributes[attr] if value is not None: if attr in old_attributes: if attr_sync_method not in ['append']: rse_module.del_rse_attribute(rse_id=rse_id, key=attr, session=session) rse_module.add_rse_attribute(rse_id=rse_id, key=attr, value=value, session=session) else: rse_module.add_rse_attribute(rse_id=rse_id, key=attr, value=value, session=session) if attr_sync_method == 'hard': for attr in missing_attributes: if attr != rse_name: rse_module.del_rse_attribute(rse_id=rse_id, key=attr, session=session) # set deleted flag to RSEs that are missing in the import data old_rses = [ old_rse['id'] for old_rse in rse_module.list_rses(session=session) ] if rse_sync_method == 'hard': for old_rse in old_rses: if old_rse not in new_rses: try: rse_module.del_rse(rse_id=old_rse, session=session) except RSEOperationNotSupported: pass
def import_rses(rses, session=None): new_rses = [] for rse_name in rses: new_rses.append(rse_name) rse = rses[rse_name] if isinstance(rse.get('rse_type'), string_types): rse['rse_type'] = RSEType.from_string(str(rse['rse_type'])) if not rse_module.rse_exists(rse_name, session=session): rse_module.add_rse(rse_name, deterministic=rse.get('deterministic'), volatile=rse.get('volatile'), city=rse.get('city'), region_code=rse.get('region_code'), country_name=rse.get('country_name'), staging_area=rse.get('staging_area'), continent=rse.get('continent'), time_zone=rse.get('time_zone'), ISP=rse.get('ISP'), rse_type=rse.get('rse_type'), latitude=rse.get('latitude'), longitude=rse.get('longitude'), ASN=rse.get('ASN'), availability=rse.get('availability'), session=session) else: rse_module.update_rse(rse_name, rse, session=session) # Protocols new_protocols = rse.get('protocols') if new_protocols: # update existing, add missing and remove left over protocols old_protocols = [{ 'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port'] } for protocol in rse_module.get_rse_protocols( rse=rse_name, session=session)['protocols']] missing_protocols = [ new_protocol for new_protocol in new_protocols if { 'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port'] } not in old_protocols ] outdated_protocols = [ new_protocol for new_protocol in new_protocols if { 'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port'] } in old_protocols ] new_protocols = [{ 'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port'] } for protocol in new_protocols] to_be_removed_protocols = [ old_protocol for old_protocol in old_protocols if old_protocol not in new_protocols ] for protocol in outdated_protocols: scheme = protocol['scheme'] port = protocol['port'] hostname = protocol['hostname'] del protocol['scheme'] del protocol['hostname'] del protocol['port'] rse_module.update_protocols(rse=rse_name, scheme=scheme, data=protocol, hostname=hostname, port=port, session=session) for protocol in missing_protocols: rse_module.add_protocol(rse=rse_name, parameter=protocol, session=session) for protocol in to_be_removed_protocols: scheme = protocol['scheme'] port = protocol['port'] hostname = protocol['hostname'] rse_module.del_protocols(rse=rse_name, scheme=scheme, port=port, hostname=hostname, session=session) # Limits old_limits = rse_module.get_rse_limits(rse=rse_name, session=session) for limit_name in ['MaxBeingDeletedFiles', 'MinFreeSpace']: limit = rse.get(limit_name) if limit: if limit_name in old_limits: rse_module.delete_rse_limit(rse=rse_name, name=limit_name, session=session) rse_module.set_rse_limits(rse=rse_name, name=limit_name, value=limit, session=session) # Attributes attributes = rse.get('attributes', {}) attributes['lfn2pfn_algorithm'] = rse.get('lfn2pfn_algorithm') attributes['verify_checksum'] = rse.get('verify_checksum') old_attributes = rse_module.list_rse_attributes(rse=rse_name, session=session) for attr in attributes: value = attributes[attr] if value is not None: if attr in old_attributes: rse_module.del_rse_attribute(rse=rse_name, key=attr, session=session) rse_module.add_rse_attribute(rse=rse_name, key=attr, value=value, session=session) # set deleted flag to RSEs that are missing in the import data old_rses = [ old_rse['rse'] for old_rse in rse_module.list_rses(session=session) ] for old_rse in old_rses: if old_rse not in new_rses: try: rse_module.del_rse(rse=old_rse, session=session) except RSEOperationNotSupported: pass