def __init__(self, bkClient=None, dm=None): """Module initialization. """ self.log = gLogger.getSubLogger("UserJobFinalization") super(UserJobFinalization, self).__init__(self.log, bkClientIn=bkClient, dm=dm) self.version = __RCSID__ self.enable = True self.defaultOutputSE = resolveSEGroup( gConfig.getValue('/Resources/StorageElementGroups/Tier1-USER', [])) self.failoverSEs = resolveSEGroup( gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover', [])) # List all parameters here self.request = None # Always allow any files specified by users self.outputDataFileMask = '' self.userOutputData = [] self.userOutputSE = '' self.userOutputPath = '' self.failoverTransfer = None self.replicateUserOutputData = False self.userPrependString = ''
def storageType(seList): """ Return the storage type: disk, tape or archive """ if not (set(se for se in seList if not dmsHelper.isSEArchive(se)) - set(['CERN-SWTEST', 'CERN-FREEZER-EOS', 'CERN-FREEZER'])) and \ set(se for se in seList if dmsHelper.isSEArchive(se)): # Only -ARCHIVE return 'Archived' tapeSEs = resolveSEGroup('Tier1-RAW') + resolveSEGroup('Tier1-RDST') if [se for se in seList if se in tapeSEs]: # Any file on Tape return 'Tape' # Others return 'Disk'
def getOption(self, switch, default=None): """ Get a specific items set by the setters """ if switch == 'SEs': # SEs have to be resolved recursively using StorageElementGroups return resolveSEGroup(self.options.get(switch, default)) value = self.options.get(switch, default) if switch in ('LFNs', 'Directory'): # Special case for getting LFNs or directories: parse the option if value == default and switch == 'Directory': value = self.options.get('LFNs', default) if not value: if not sys.stdin.isatty(): # If the input file is a pipe, no need to specify it self.setLFNsFromTerm() value = self.options.get('LFNs', default) if value: # Parse the LFNs out of the "LFNs" option list value = self.getLFNsFromList(value, directories=switch == 'Directory') if value and self.setLastFile and switch == 'LFNs': # Storethe list of LFNs in a temporary file gLogger.always("Got %d LFNs" % len(value)) if self.setLastFile != self.lastFile: self.setLastFile = False with open(self.lastFile, 'w') as tmpFile: tmpFile.write('\n'.join(sorted(value))) if isinstance(value, set): # Return a sorted list from a set value = sorted(value) return value
def _Transformation__checkSEs(self, seList): # This test allows to set some parameters empty if seList == []: return S_OK() if resolveSEGroup(seList): return S_OK() gLogger.error("Some SEs are unknown in %s" % ','.join(seList)) return S_ERROR("Some StorageElements not known")
def main(): from DIRAC.Core.Base import Script Script.registerSwitch('', 'Path=', ' Path to search for') Script.registerSwitch( '', 'SE=', ' (comma-separated list of) SEs/SE-groups to be searched') Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() import DIRAC from DIRAC import gLogger from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup path = '/' seList = None for opt, val in Script.getUnprocessedSwitches(): if opt == 'Path': path = val elif opt == 'SE': seList = resolveSEGroup(val.split(',')) if seList: args.append("SE=%s" % ','.join(seList)) fc = FileCatalog() result = fc.getMetadataFields() if not result['OK']: gLogger.error('Can not access File Catalog:', result['Message']) DIRAC.exit(-1) typeDict = result['Value']['FileMetaFields'] typeDict.update(result['Value']['DirectoryMetaFields']) # Special meta tags typeDict.update(FILE_STANDARD_METAKEYS) if len(args) < 1: print("Error: No argument provided\n%s:" % Script.scriptName) gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict)) Script.showHelp(exitCode=1) mq = MetaQuery(typeDict=typeDict) result = mq.setMetaQuery(args) if not result['OK']: gLogger.error("Illegal metaQuery:", result['Message']) DIRAC.exit(-1) metaDict = result['Value'] path = metaDict.pop('Path', path) result = fc.findFilesByMetadata(metaDict, path) if not result['OK']: gLogger.error('Can not access File Catalog:', result['Message']) DIRAC.exit(-1) lfnList = sorted(result['Value']) gLogger.notice('\n'.join(lfn for lfn in lfnList))
pass if switch[0].lower() in ("m", "mute"): mute = True if switch[0].lower() in ("s", "site"): site = switch[1] # from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup ses = resolveSEGroup(ses) diracAdmin = DiracAdmin() errorList = [] setup = gConfig.getValue('/DIRAC/Setup', '') if not setup: print('ERROR: Could not contact Configuration Service') DIRAC.exit(2) res = getProxyInfo() if not res['OK']: gLogger.error('Failed to get proxy information', res['Message']) DIRAC.exit(2) userName = res['Value'].get('username') if not userName: gLogger.error('Failed to get username for proxy')
def main(): read = True write = True check = True remove = True site = '' mute = False Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element") Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element") Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element") Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element") Script.registerSwitch("a", "All", " Ban all access to the storage element") Script.registerSwitch("m", "Mute", " Do not send email") Script.registerSwitch( "S:", "Site=", " Ban all SEs associate to site (note that if writing is allowed, check is always allowed)") Script.parseCommandLine(ignoreErrors=True) ses = Script.getPositionalArgs() for switch in Script.getUnprocessedSwitches(): if switch[0].lower() in ("r", "banread"): write = False check = False remove = False if switch[0].lower() in ("w", "banwrite"): read = False check = False remove = False if switch[0].lower() in ("k", "bancheck"): read = False write = False remove = False if switch[0].lower() in ("v", "banremove"): read = False write = False check = False if switch[0].lower() in ("a", "all"): pass if switch[0].lower() in ("m", "mute"): mute = True if switch[0].lower() in ("s", "site"): site = switch[1] # from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup ses = resolveSEGroup(ses) diracAdmin = DiracAdmin() errorList = [] setup = gConfig.getValue('/DIRAC/Setup', '') if not setup: print('ERROR: Could not contact Configuration Service') DIRAC.exit(2) res = getProxyInfo() if not res['OK']: gLogger.error('Failed to get proxy information', res['Message']) DIRAC.exit(2) userName = res['Value'].get('username') if not userName: gLogger.error('Failed to get username for proxy') DIRAC.exit(2) if site: res = getSites() if not res['OK']: gLogger.error(res['Message']) DIRAC.exit(-1) if site not in res['Value']: gLogger.error('The provided site (%s) is not known.' % site) DIRAC.exit(-1) ses.extend(res['Value']['SE'].replace(' ', '').split(',')) if not ses: gLogger.error('There were no SEs provided') DIRAC.exit(-1) readBanned = [] writeBanned = [] checkBanned = [] removeBanned = [] resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus(ses, "StorageElement") if not res['OK']: gLogger.error("Storage Element %s does not exist" % ses) DIRAC.exit(-1) reason = 'Forced with dirac-admin-ban-se by %s' % userName for se, seOptions in res['Value'].items(): resW = resC = resR = {'OK': False} # Eventually, we will get rid of the notion of InActive, as we always write Banned. if read and 'ReadAccess' in seOptions: if not seOptions['ReadAccess'] in ['Active', 'Degraded', 'Probing']: gLogger.notice('Read option for %s is %s, instead of %s' % (se, seOptions['ReadAccess'], ['Active', 'Degraded', 'Probing'])) gLogger.notice('Try specifying the command switches') else: resR = resourceStatus.setElementStatus(se, 'StorageElement', 'ReadAccess', 'Banned', reason, userName) # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) if not resR['OK']: gLogger.error('Failed to update %s read access to Banned' % se) else: gLogger.notice('Successfully updated %s read access to Banned' % se) readBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if write and 'WriteAccess' in seOptions: if not seOptions['WriteAccess'] in ['Active', 'Degraded', 'Probing']: gLogger.notice('Write option for %s is %s, instead of %s' % (se, seOptions['WriteAccess'], ['Active', 'Degraded', 'Probing'])) gLogger.notice('Try specifying the command switches') else: resW = resourceStatus.setElementStatus(se, 'StorageElement', 'WriteAccess', 'Banned', reason, userName) # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) if not resW['OK']: gLogger.error("Failed to update %s write access to Banned" % se) else: gLogger.notice("Successfully updated %s write access to Banned" % se) writeBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if check and 'CheckAccess' in seOptions: if not seOptions['CheckAccess'] in ['Active', 'Degraded', 'Probing']: gLogger.notice('Check option for %s is %s, instead of %s' % (se, seOptions['CheckAccess'], ['Active', 'Degraded', 'Probing'])) gLogger.notice('Try specifying the command switches') else: resC = resourceStatus.setElementStatus(se, 'StorageElement', 'CheckAccess', 'Banned', reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC['OK']: gLogger.error("Failed to update %s check access to Banned" % se) else: gLogger.notice("Successfully updated %s check access to Banned" % se) checkBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if remove and 'RemoveAccess' in seOptions: if not seOptions['RemoveAccess'] in ['Active', 'Degraded', 'Probing']: gLogger.notice('Remove option for %s is %s, instead of %s' % (se, seOptions['RemoveAccess'], ['Active', 'Degraded', 'Probing'])) gLogger.notice('Try specifying the command switches') else: resC = resourceStatus.setElementStatus(se, 'StorageElement', 'RemoveAccess', 'Banned', reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC['OK']: gLogger.error("Failed to update %s remove access to Banned" % se) else: gLogger.notice("Successfully updated %s remove access to Banned" % se) removeBanned.append(se) if not(resR['OK'] or resW['OK'] or resC['OK']): DIRAC.exit(-1) if not (writeBanned or readBanned or checkBanned or removeBanned): gLogger.notice("No storage elements were banned") DIRAC.exit(-1) if mute: gLogger.notice('Email is muted by script switch') DIRAC.exit(0) subject = '%s storage elements banned for use' % len(writeBanned + readBanned + checkBanned + removeBanned) addressPath = 'EMail/Production' address = Operations().getValue(addressPath, '') body = '' if read: body = "%s\n\nThe following storage elements were banned for reading:" % body for se in readBanned: body = "%s\n%s" % (body, se) if write: body = "%s\n\nThe following storage elements were banned for writing:" % body for se in writeBanned: body = "%s\n%s" % (body, se) if check: body = "%s\n\nThe following storage elements were banned for check access:" % body for se in checkBanned: body = "%s\n%s" % (body, se) if remove: body = "%s\n\nThe following storage elements were banned for remove access:" % body for se in removeBanned: body = "%s\n%s" % (body, se) if not address: gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) DIRAC.exit(0) res = diracAdmin.sendMail(address, subject, body) gLogger.notice('Notifying %s' % address) if res['OK']: gLogger.notice(res['Value']) else: gLogger.notice(res['Message']) DIRAC.exit(0)
write = False check = False if switch[0].lower() == "m" or switch[0].lower() == "mute": mute = True if switch[0] == "S" or switch[0].lower() == "site": site = switch[1] # from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC import gConfig, gLogger from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup ses = resolveSEGroup( ses ) diracAdmin = DiracAdmin() exitCode = 0 errorList = [] setup = gConfig.getValue( '/DIRAC/Setup', '' ) if not setup: print 'ERROR: Could not contact Configuration Service' exitCode = 2 DIRAC.exit( exitCode ) res = getProxyInfo() if not res[ 'OK' ]: gLogger.error( 'Failed to get proxy information', res[ 'Message' ] ) DIRAC.exit( 2 ) userName = res['Value'].get( 'username' )
def main(): Script.registerSwitch("", "Path=", " Path to search for") Script.registerSwitch( "", "SE=", " (comma-separated list of) SEs/SE-groups to be searched") # Registering arguments will automatically add their description to the help menu Script.registerArgument( [ "metaspec: metadata index specification (of the form: " '"meta=value" or "meta<value", "meta!=value", etc.)' ], mandatory=False, ) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() import DIRAC from DIRAC import gLogger from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup path = "/" seList = None for opt, val in Script.getUnprocessedSwitches(): if opt == "Path": path = val elif opt == "SE": seList = resolveSEGroup(val.split(",")) if seList: args.append("SE=%s" % ",".join(seList)) fc = FileCatalog() result = fc.getMetadataFields() if not result["OK"]: gLogger.error("Can not access File Catalog:", result["Message"]) DIRAC.exit(-1) typeDict = result["Value"]["FileMetaFields"] typeDict.update(result["Value"]["DirectoryMetaFields"]) # Special meta tags typeDict.update(FILE_STANDARD_METAKEYS) if len(args) < 1: print("Error: No argument provided\n%s:" % Script.scriptName) gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict)) Script.showHelp(exitCode=1) mq = MetaQuery(typeDict=typeDict) result = mq.setMetaQuery(args) if not result["OK"]: gLogger.error("Illegal metaQuery:", result["Message"]) DIRAC.exit(-1) metaDict = result["Value"] path = metaDict.pop("Path", path) result = fc.findFilesByMetadata(metaDict, path) if not result["OK"]: gLogger.error("Can not access File Catalog:", result["Message"]) DIRAC.exit(-1) lfnList = sorted(result["Value"]) gLogger.notice("\n".join(lfn for lfn in lfnList))
def main(): read = True write = True check = True remove = True sites = [] mute = False Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element") Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element") Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element") Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element") Script.registerSwitch("a", "All", " Ban all access to the storage element") Script.registerSwitch("m", "Mute", " Do not send email") Script.registerSwitch( "S:", "Site=", " Ban all SEs associate to site (note that if writing is allowed, check is always allowed)" ) # Registering arguments will automatically add their description to the help menu Script.registerArgument( ["seGroupList: list of SEs or comma-separated SEs"]) switches, ses = Script.parseCommandLine(ignoreErrors=True) for switch in switches: if switch[0].lower() in ("r", "banread"): write = False check = False remove = False if switch[0].lower() in ("w", "banwrite"): read = False check = False remove = False if switch[0].lower() in ("k", "bancheck"): read = False write = False remove = False if switch[0].lower() in ("v", "banremove"): read = False write = False check = False if switch[0].lower() in ("a", "all"): pass if switch[0].lower() in ("m", "mute"): mute = True if switch[0].lower() in ("s", "site"): sites = switch[1].split(",") # from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup, DMSHelpers ses = resolveSEGroup(ses) diracAdmin = DiracAdmin() setup = gConfig.getValue("/DIRAC/Setup", "") if not setup: print("ERROR: Could not contact Configuration Service") DIRAC.exit(2) res = getProxyInfo() if not res["OK"]: gLogger.error("Failed to get proxy information", res["Message"]) DIRAC.exit(2) userName = res["Value"].get("username") if not userName: gLogger.error("Failed to get username for proxy") DIRAC.exit(2) for site in sites: res = DMSHelpers().getSEsForSite(site) if not res["OK"]: gLogger.error(res["Message"], site) DIRAC.exit(-1) ses.extend(res["Value"]) if not ses: gLogger.error("There were no SEs provided") DIRAC.exit(-1) readBanned = [] writeBanned = [] checkBanned = [] removeBanned = [] resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus(ses, "StorageElement") if not res["OK"]: gLogger.error("Storage Element %s does not exist" % ses) DIRAC.exit(-1) reason = "Forced with dirac-admin-ban-se by %s" % userName for se, seOptions in res["Value"].items(): resW = resC = resR = {"OK": False} # Eventually, we will get rid of the notion of InActive, as we always write Banned. if read and "ReadAccess" in seOptions: if seOptions["ReadAccess"] == "Banned": gLogger.notice("Read access already banned", se) resR["OK"] = True elif not seOptions["ReadAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Read option for %s is %s, instead of %s" % (se, seOptions["ReadAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resR = resourceStatus.setElementStatus(se, "StorageElement", "ReadAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) if not resR["OK"]: gLogger.error("Failed to update %s read access to Banned" % se) else: gLogger.notice( "Successfully updated %s read access to Banned" % se) readBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if write and "WriteAccess" in seOptions: if seOptions["WriteAccess"] == "Banned": gLogger.notice("Write access already banned", se) resW["OK"] = True elif not seOptions["WriteAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Write option for %s is %s, instead of %s" % (se, seOptions["WriteAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resW = resourceStatus.setElementStatus(se, "StorageElement", "WriteAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) if not resW["OK"]: gLogger.error( "Failed to update %s write access to Banned" % se) else: gLogger.notice( "Successfully updated %s write access to Banned" % se) writeBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if check and "CheckAccess" in seOptions: if seOptions["CheckAccess"] == "Banned": gLogger.notice("Check access already banned", se) resC["OK"] = True elif not seOptions["CheckAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Check option for %s is %s, instead of %s" % (se, seOptions["CheckAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resC = resourceStatus.setElementStatus(se, "StorageElement", "CheckAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC["OK"]: gLogger.error( "Failed to update %s check access to Banned" % se) else: gLogger.notice( "Successfully updated %s check access to Banned" % se) checkBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if remove and "RemoveAccess" in seOptions: if seOptions["RemoveAccess"] == "Banned": gLogger.notice("Remove access already banned", se) resC["OK"] = True elif not seOptions["RemoveAccess"] in [ "Active", "Degraded", "Probing" ]: gLogger.notice("Remove option for %s is %s, instead of %s" % (se, seOptions["RemoveAccess"], ["Active", "Degraded", "Probing"])) gLogger.notice("Try specifying the command switches") else: resC = resourceStatus.setElementStatus(se, "StorageElement", "RemoveAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC["OK"]: gLogger.error( "Failed to update %s remove access to Banned" % se) else: gLogger.notice( "Successfully updated %s remove access to Banned" % se) removeBanned.append(se) if not (resR["OK"] or resW["OK"] or resC["OK"]): DIRAC.exit(-1) if not (writeBanned or readBanned or checkBanned or removeBanned): gLogger.notice("No storage elements were banned") DIRAC.exit(-1) if mute: gLogger.notice("Email is muted by script switch") DIRAC.exit(0) subject = "%s storage elements banned for use" % len( writeBanned + readBanned + checkBanned + removeBanned) addressPath = "EMail/Production" address = Operations().getValue(addressPath, "") body = "" if read: body = "%s\n\nThe following storage elements were banned for reading:" % body for se in readBanned: body = "%s\n%s" % (body, se) if write: body = "%s\n\nThe following storage elements were banned for writing:" % body for se in writeBanned: body = "%s\n%s" % (body, se) if check: body = "%s\n\nThe following storage elements were banned for check access:" % body for se in checkBanned: body = "%s\n%s" % (body, se) if remove: body = "%s\n\nThe following storage elements were banned for remove access:" % body for se in removeBanned: body = "%s\n%s" % (body, se) if not address: gLogger.notice( "'%s' not defined in Operations, can not send Mail\n" % addressPath, body) DIRAC.exit(0) res = diracAdmin.sendMail(address, subject, body) gLogger.notice("Notifying %s" % address) if res["OK"]: gLogger.notice(res["Value"]) else: gLogger.notice(res["Message"]) DIRAC.exit(0)
def main(): read = False write = False check = False remove = False site = '' mute = False Script.registerSwitch("r", "AllowRead", " Allow only reading from the storage element") Script.registerSwitch("w", "AllowWrite", " Allow only writing to the storage element") Script.registerSwitch( "k", "AllowCheck", " Allow only check access to the storage element") Script.registerSwitch( "v", "AllowRemove", " Allow only remove access to the storage element") Script.registerSwitch("a", "All", " Allow all access to the storage element") Script.registerSwitch("m", "Mute", " Do not send email") Script.registerSwitch("S:", "Site=", " Allow all SEs associated to site") Script.parseCommandLine(ignoreErrors=True) ses = Script.getPositionalArgs() for switch in Script.getUnprocessedSwitches(): if switch[0].lower() in ("r", "allowread"): read = True if switch[0].lower() in ("w", "allowwrite"): write = True if switch[0].lower() in ("k", "allowcheck"): check = True if switch[0].lower() in ("v", "allowremove"): remove = True if switch[0].lower() in ("a", "all"): read = True write = True check = True remove = True if switch[0].lower() in ("m", "mute"): mute = True if switch[0].lower() in ("s", "site"): site = switch[1] # imports from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus if not (read or write or check or remove): # No switch was specified, means we need all of them gLogger.notice( "No option given, all accesses will be allowed if they were not") read = True write = True check = True remove = True ses = resolveSEGroup(ses) diracAdmin = DiracAdmin() errorList = [] setup = gConfig.getValue('/DIRAC/Setup', '') if not setup: print('ERROR: Could not contact Configuration Service') DIRAC.exit(2) res = getProxyInfo() if not res['OK']: gLogger.error('Failed to get proxy information', res['Message']) DIRAC.exit(2) userName = res['Value'].get('username') if not userName: gLogger.error('Failed to get username for proxy') DIRAC.exit(2) if site: res = getSites() if not res['OK']: gLogger.error(res['Message']) DIRAC.exit(-1) if site not in res['Value']: gLogger.error('The provided site (%s) is not known.' % site) DIRAC.exit(-1) ses.extend(res['Value']['SE'].replace(' ', '').split(',')) if not ses: gLogger.error('There were no SEs provided') DIRAC.exit() STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"] ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded"] statusAllowedDict = {} for statusType in STATUS_TYPES: statusAllowedDict[statusType] = [] statusFlagDict = {} statusFlagDict['ReadAccess'] = read statusFlagDict['WriteAccess'] = write statusFlagDict['CheckAccess'] = check statusFlagDict['RemoveAccess'] = remove resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus(ses, "StorageElement") if not res['OK']: gLogger.error('Storage Element %s does not exist' % ses) DIRAC.exit(-1) reason = 'Forced with dirac-admin-allow-se by %s' % userName for se, seOptions in res['Value'].items(): # InActive is used on the CS model, Banned is the equivalent in RSS for statusType in STATUS_TYPES: if statusFlagDict[statusType]: if seOptions.get(statusType) == "Active": gLogger.notice('%s status of %s is already Active' % (statusType, se)) continue if statusType in seOptions: if not seOptions[statusType] in ALLOWED_STATUSES: gLogger.notice( '%s option for %s is %s, instead of %s' % (statusType, se, seOptions['ReadAccess'], ALLOWED_STATUSES)) gLogger.notice('Try specifying the command switches') else: resR = resourceStatus.setElementStatus( se, "StorageElement", statusType, 'Active', reason, userName) if not resR['OK']: gLogger.fatal( "Failed to update %s %s to Active, exit -" % (se, statusType), resR['Message']) DIRAC.exit(-1) else: gLogger.notice( "Successfully updated %s %s to Active" % (se, statusType)) statusAllowedDict[statusType].append(se) totalAllowed = 0 totalAllowedSEs = [] for statusType in STATUS_TYPES: totalAllowed += len(statusAllowedDict[statusType]) totalAllowedSEs += statusAllowedDict[statusType] totalAllowedSEs = list(set(totalAllowedSEs)) if not totalAllowed: gLogger.info("No storage elements were allowed") DIRAC.exit(-1) if mute: gLogger.notice('Email is muted by script switch') DIRAC.exit(0) subject = '%s storage elements allowed for use' % len(totalAllowedSEs) addressPath = 'EMail/Production' address = Operations().getValue(addressPath, '') body = '' if read: body = "%s\n\nThe following storage elements were allowed for reading:" % body for se in statusAllowedDict['ReadAccess']: body = "%s\n%s" % (body, se) if write: body = "%s\n\nThe following storage elements were allowed for writing:" % body for se in statusAllowedDict['WriteAccess']: body = "%s\n%s" % (body, se) if check: body = "%s\n\nThe following storage elements were allowed for checking:" % body for se in statusAllowedDict['CheckAccess']: body = "%s\n%s" % (body, se) if remove: body = "%s\n\nThe following storage elements were allowed for removing:" % body for se in statusAllowedDict['RemoveAccess']: body = "%s\n%s" % (body, se) if not address: gLogger.notice( "'%s' not defined in Operations, can not send Mail\n" % addressPath, body) DIRAC.exit(0) res = diracAdmin.sendMail(address, subject, body) gLogger.notice('Notifying %s' % address) if res['OK']: gLogger.notice(res['Value']) else: gLogger.notice(res['Message']) DIRAC.exit(0)
Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS from DIRAC import gLogger from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup path = '/' seList = None for opt, val in Script.getUnprocessedSwitches(): if opt == 'Path': path = val elif opt == 'SE': seList = resolveSEGroup(val.split(',')) if seList: args.append("SE=%s" % ','.join(seList)) fc = FileCatalog() result = fc.getMetadataFields() if not result['OK']: gLogger.error('Can not access File Catalog:', result['Message']) DIRAC.exit(-1) typeDict = result['Value']['FileMetaFields'] typeDict.update(result['Value']['DirectoryMetaFields']) # Special meta tags typeDict.update(FILE_STANDARD_METAKEYS) if len(args) < 1: print "Error: No argument provided\n%s:" % Script.scriptName
def getDestinationSEList(outputSE, site, outputmode='Any', run=None): """ Evaluate the output SE list from a workflow and return the concrete list of SEs to upload output data. """ if outputmode.lower() not in ('any', 'local', 'run'): raise RuntimeError("Unexpected outputmode") if outputmode.lower() == 'run': gLogger.verbose( "Output mode set to 'run', thus ignoring site parameter") if not run: raise RuntimeError("Expected runNumber") try: run = long(run) except ValueError as ve: raise RuntimeError("Expected runNumber as a number: %s" % ve) gLogger.debug("RunNumber = %d" % run) from LHCbDIRAC.TransformationSystem.Client.TransformationClient import TransformationClient runDestination = TransformationClient().getDestinationForRun(run) if not runDestination['OK'] or run not in runDestination['Value']: raise RuntimeError("Issue getting destinationForRun (%d): " % run + runDestination.get('Message', 'unknown run')) site = runDestination['Value'][run] gLogger.verbose("Site set to %s for run %d" % (site, run)) outputmode = 'Local' # Add output SE defined in the job description gLogger.info('Resolving workflow output SE description: %s' % outputSE) # Check if the SE is defined explicitly for the site prefix = site.split('.')[0] country = site.split('.')[-1] # Concrete SE name result = gConfig.getOptions('/Resources/StorageElements/' + outputSE) if result['OK']: gLogger.info('Found concrete SE %s' % outputSE) return [outputSE] # Get local SEs localSEs = getSEsForSite(site) if not localSEs['OK']: raise RuntimeError(localSEs['Message']) localSEs = localSEs['Value'] gLogger.verbose("Local SE list is: %s" % (localSEs)) # There is an alias defined for this Site associatedSEs = gConfig.getValue( '/Resources/Sites/%s/%s/AssociatedSEs/%s' % (prefix, site, outputSE), []) if associatedSEs: associatedSEs = _setLocalFirst(associatedSEs, localSEs) gLogger.info("Found associated SE %s for site %s" % (associatedSEs, site)) return associatedSEs groupSEs = resolveSEGroup(outputSE) if not groupSEs: raise RuntimeError("Failed to resolve SE " + outputSE) gLogger.verbose("Group SE list is: %s" % (groupSEs)) # Find a local SE or an SE considered as local because the country is associated to it if outputmode.lower() == "local": # First, check if one SE in the group is local for se in localSEs: if se in groupSEs: gLogger.info("Found eligible local SE: %s" % (se)) return [se] # Final check for country associated SE assignedCountry = country while True: # check if country is already one with associated SEs section = '/Resources/Countries/%s/AssociatedSEs/%s' % ( assignedCountry, outputSE) associatedSEs = gConfig.getValue(section, []) if associatedSEs: associatedSEs = _setLocalFirst(associatedSEs, localSEs) gLogger.info('Found associated SEs %s in %s' % (associatedSEs, section)) return associatedSEs gLogger.verbose("/Resources/Countries/%s/AssignedTo" % assignedCountry) opt = gConfig.getOption("/Resources/Countries/%s/AssignedTo" % assignedCountry) if opt['OK'] and opt['Value']: assignedCountry = opt['Value'] else: # No associated SE and no assigned country, give up raise RuntimeError( "Could not establish associated SE nor assigned country for country %s" % assignedCountry) # For collective Any and All modes return the whole group # Make sure that local SEs are passing first orderedSEs = _setLocalFirst(groupSEs, localSEs) gLogger.info('Found SEs, local first: %s' % orderedSEs) return orderedSEs
def executeForVO(self, vo): """ Execute one SE and user synchronisation cycle for a VO. :param str vo: Virtual organisation name. :return: S_OK or S_ERROR :rtype: dict """ valid_protocols = ["srm", "gsiftp", "davs", "https", "root"] default_email = None try: try: client = Client(account="root", auth_type="userpass") except Exception as err: self.log.info( "Login to Rucio as root with password failed. Will try host cert/key", str(err)) certKeyTuple = Locations.getHostCertificateAndKeyLocation() if not certKeyTuple: self.log.error("Hostcert/key location not set") return S_ERROR("Hostcert/key location not set") hostcert, hostkey = certKeyTuple self.log.info("Logging in with a host cert/key pair:") self.log.debug("account: ", self.clientConfig[vo]["privilegedAccount"]) self.log.debug("rucio host: ", self.clientConfig[vo]["rucioHost"]) self.log.debug("auth host: ", self.clientConfig[vo]["authHost"]) self.log.debug("CA cert path: ", self.caCertPath) self.log.debug("Cert location: ", hostcert) self.log.debug("Key location: ", hostkey) self.log.debug("VO: ", vo) client = Client( account=self.clientConfig[vo]["privilegedAccount"], rucio_host=self.clientConfig[vo]["rucioHost"], auth_host=self.clientConfig[vo]["authHost"], ca_cert=self.caCertPath, auth_type="x509", creds={ "client_cert": hostcert, "client_key": hostkey }, timeout=600, user_agent="rucio-clients", vo=vo, ) self.log.info("Rucio client instantiated for VO:", vo) # Get the storage elements from Dirac Configuration and create them in Rucio newRSE = False self.log.info("Synchronizing SEs for VO ", vo) result = getStorageElements(vo) if result["OK"]: rses = [rse["rse"] for rse in client.list_rses()] for se in result["Value"]: if se not in rses: # The SE doesn't exist. Will create it newRSE = True self.log.info( "Rucio Storage Element does not exist and will be created:", se) try: client.add_rse(rse=se, deterministic=True, volatile=False) except Exception as err: self.log.error( "Cannot create RSE", "[RSE: %s, Error: %s]" % (se, str(err))) continue # Add RSE attributes for the new RSE ret = gConfig.getOptionsDict( "Resources/FTSEndpoints/FTS3") ftsList = "" if ret["OK"]: ftsList = ",".join(ret["Value"].values()) dictRSEAttributes = { "naming_convention": "BelleII", "ANY": True, "fts": ftsList } for key in dictRSEAttributes: self.log.info( "Setting RSE attributes", "[RSE: %s, Attr. name: %s, Value: %s]" % (se, key, dictRSEAttributes[key]), ) client.add_rse_attribute( se, key, value=dictRSEAttributes[key]) client.set_local_account_limit("root", se, 100000000000000000) # Create the protocols try: protocols = client.get_protocols(se) except RSEProtocolNotSupported as err: self.log.info("Cannot get protocols for", "[RSE %s, Error: %s]" % (se, str(err))) protocols = [] existing_protocols = [] for prot in protocols: existing_protocols.append( (str(prot["scheme"]), str(prot["hostname"]), str(prot["port"]), str(prot["prefix"]))) protocols_to_create = [] for params in result["Value"][se]: prot = ( str(params["scheme"]), str(params["hostname"]), str(params["port"]), str(params["prefix"]), ) protocols_to_create.append(prot) if prot not in existing_protocols and prot[ 0] in valid_protocols: # The protocol defined in Dirac does not exist in Rucio. Will be created self.log.info( "Will create new protocol:", "%s://%s:%s%s on %s" % (params["scheme"], params["hostname"], params["port"], params["prefix"], se), ) try: client.add_protocol(rse=se, params=params) except Duplicate as err: self.log.info( "Protocol already exists on", "[RSE: %s, schema:%s]" % (se, params["scheme"])) except Exception as err: self.log.error( "Cannot create protocol on RSE", "[RSE: %s, Error: %s]" % (se, str(err))) else: update = False for protocol in protocols: if prot == ( str(protocol["scheme"]), str(protocol["hostname"]), str(protocol["port"]), str(protocol["prefix"]), ): # Check if the protocol defined in Dirac has the same priority as the one defined in Rucio for domain in ["lan", "wan"]: for activity in [ "read", "write", "delete" ]: if (params["domains"][domain] [activity] != protocol["domains"][domain] [activity]): update = True break if (params["domains"]["wan"] ["third_party_copy"] != protocol["domains"]["wan"] ["third_party_copy"]): update = True if update: data = { "prefix": params["prefix"], "read_lan": params["domains"]["lan"]["read"], "read_wan": params["domains"]["wan"]["read"], "write_lan": params["domains"]["lan"]["write"], "write_wan": params["domains"]["wan"]["write"], "delete_lan": params["domains"]["lan"]["delete"], "delete_wan": params["domains"]["wan"]["delete"], "third_party_copy": params["domains"]["wan"]["write"], } self.log.info( "Will update protocol:", "%s://%s:%s%s on %s" % ( params["scheme"], params["hostname"], params["port"], params["prefix"], se, ), ) client.update_protocols( rse=se, scheme=params["scheme"], data=data, hostname=params["hostname"], port=params["port"], ) for prot in existing_protocols: if prot not in protocols_to_create: self.log.info( "Will delete protocol:", "%s://%s:%s%s on %s" % (prot[0], prot[1], prot[2], prot[3], se)) client.delete_protocols(se, scheme=prot[0], hostname=prot[1], port=prot[2]) else: self.log.error("Cannot get SEs:", result["Message"]) # If new RSE added, add distances rses = [rse["rse"] for rse in client.list_rses()] if newRSE: self.log.info("Adding distances") for src_rse, dest_rse in permutations(rses, r=2): try: client.add_distance(src_rse, dest_rse, { "ranking": 1, "distance": 10 }) except Exception as err: self.log.error( "Cannot add distance for", "Source RSE: %s, Dest RSE: %s, Error:%s" % (src_rse, dest_rse, str(err)), ) # Collect the shares from Dirac Configuration and create them in Rucio self.log.info("Synchronizing shares") result = Operations().getOptionsDict("Production/SEshares") if result["OK"]: rseDict = result["Value"] for rse in rses: try: self.log.info("Setting productionSEshare for", "[RSE: %s : Share: %s", rse, rseDict.get(rse, 0)) client.add_rse_attribute(rse, "productionSEshare", rseDict.get(rse, 0)) except Exception as err: self.log.error( "Cannot create productionSEshare for RSE:", rse) else: self.log.error("Cannot get SEs", result["Message"]) result = Operations().getSections("Shares") if result["OK"]: for dataLevel in result["Value"]: result = Operations().getOptionsDict("Shares/%s" % dataLevel) if not result["OK"]: self.log.error("Cannot get SEs:" % result["Message"]) continue rseDict = result["Value"] for rse in rses: try: self.log.info( "Setting", "%sShare for %s : %s" % (dataLevel, rse, rseDict.get(rse, 0))) client.add_rse_attribute(rse, "%sShare" % dataLevel, rseDict.get(rse, 0)) except Exception as err: self.log.error("Cannot create share:", "%sShare for %s", dataLevel, rse) else: self.log.error("Cannot get shares:", result["Message"]) # Create the RSE attribute PrimaryDataSE and OccupancyLFN result = gConfig.getValue( "Resources/StorageElementGroups/PrimarySEs") result = getStorageElements(vo) if result["OK"]: allSEs = result["Value"] primarySEs = resolveSEGroup("PrimarySEs", allSEs) self.log.info("Will set primarySEs flag to:", str(primarySEs)) for rse in rses: if rse in allSEs: storage = StorageElement(rse) if not storage.valid: self.log.warn( "Storage element is not valid. Skipped RSE:", rse) continue occupancyLFN = storage.options.get("OccupancyLFN") try: client.add_rse_attribute(rse, "OccupancyLFN", occupancyLFN) except Exception as err: self.log.error( "Cannot create RSE attribute OccupancyLFN for", "[RSE: %s, Error: %s]" % (rse, str(err))) if rse in primarySEs: try: client.add_rse_attribute(rse, "PrimaryDataSE", True) except Exception as err: self.log.error( "Cannot create RSE attribute PrimaryDataSE for", "[RSE: %s, Error: %s]" % (rse, str(err)), ) else: try: client.delete_rse_attribute(rse, "PrimaryDataSE") except RSEAttributeNotFound: pass except Exception as err: self.log.error( "Cannot remove RSE attribute PrimaryDataSE for", "[RSE: %s, Error: %s]" % (rse, str(err)), ) self.log.info("RSEs synchronized for VO: ", vo) # Collect the user accounts from Dirac Configuration and create user accounts in Rucio self.log.info("Synchronizing accounts for VO", vo) listAccounts = [ str(acc["account"]) for acc in client.list_accounts() ] listScopes = [str(scope) for scope in client.list_scopes()] dnMapping = {} diracUsers = getUsersInVO(vo) self.log.debug(" Will consider following Dirac users for", "[VO: %s, Dirac users: %s]" % (vo, diracUsers)) for account in diracUsers: dn = getUserOption(account, "DN") email = getUserOption(account, "Email") dnMapping[dn] = email if account not in listAccounts: self.log.info("Will create account with associated DN ", "[account: %s, DN: %s]" % (account, dn)) try: client.add_account(account, "USER", email) listAccounts.append(account) except Exception as err: self.log.error( "Cannot create account", "[account: %s, Error: %s]" % (account, str(err))) try: client.add_identity(account=account, identity=dn, authtype="X509", email=email, default=True) except Exception as err: self.log.error( "Cannot add identity for account", "[Identity: dn=%s, account:%s, Error: %s]" % (dn, account, str(err)), ) self.log.error( " Account/identity skipped (it will not be created in Rucio)", "[%s/%s]" % (account, dn)) continue for rse in rses: client.set_local_account_limit(account, rse, 1000000000000000) else: try: client.add_identity(account=account, identity=dn, authtype="X509", email=email, default=True) except Duplicate: pass except Exception as err: self.log.error( "Cannot create identity for account", "[DN: %s, account: %s, Error: %s]" % (dn, account, str(err)), ) scope = "user." + account if scope not in listScopes: try: self.log.info("Will create a scope", "[Scope: %s]" % scope) client.add_scope(account, scope) self.log.info("Scope successfully added", "[Scope: %s]" % scope) except Exception as err: self.log.error( "Cannot create a scope", "[Scope: %s, Error: %s]" % (scope, str(err))) # Collect the group accounts from Dirac Configuration and create service accounts in Rucio result = getGroupsForVO(vo) if result["OK"]: groups = result["Value"] self.log.debug(" Will consider following Dirac groups for", "[%s VO: %s]" % (vo, groups)) else: groups = [] self.log.debug("No Dirac groups for", "%s VO " % vo) self.log.debug("No Rucio service accounts will be created") for group in groups: if group not in listAccounts: self.log.info( "Will create SERVICE account for Dirac group:", str(group)) try: client.add_account(group, "SERVICE", None) listAccounts.append(group) except Exception as err: self.log.error( "Cannot create SERVICE account for", "[group: %s, Error: %s]" % (group, str(err))) for rse in rses: client.set_local_account_limit(account, rse, 1000000000000000) for dn in getDNsInGroup(group): try: client.add_identity(account=group, identity=dn, authtype="X509", email=dnMapping.get( dn, default_email)) except Duplicate: pass except Exception as err: self.log.error( "Cannot create identity for account", "[identity %s, account %s, Error: %s]" % (dn, group, str(err)), ) self.log.error(format_exc()) # Collect the group accounts from Dirac Configuration and create service accounts in Rucio result = getHosts() if not result["OK"]: self.log.error("Cannot get host accounts:", "%s" % result["Message"]) else: hosts = result["Value"] for host in hosts: dn = getHostOption(host, "DN") email = dnMapping.get(dn, default_email) try: client.add_identity(account="dirac_srv", identity=dn, authtype="X509", email=email) except Duplicate: pass except Exception as err: self.log.error( "Cannot create identity for account dirac_srv:", "[DN: %s, Error: %s]" % (dn, str(err))) self.log.error(format_exc()) return S_OK() except Exception as exc: self.log.exception("Synchronisation for VO failed. VO skipped ", "VO=%s" % vo, lException=exc) return S_ERROR(str(format_exc()))
def execute(): """ Parse the options and execute the script """ bkQuery = dmScript.getBKQuery() fileType = bkQuery.getFileTypeList() if not set(fileType) & {'FULL.DST', 'RDST', 'SDST'}: gLogger.error("Please provide a reconstruction BK path") DIRAC.exit(1) from LHCbDIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC.Core.Utilities.List import breakListIntoChunks from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers, resolveSEGroup bk = BookkeepingClient() tr = TransformationClient() dm = DataManager() dmsHelper = DMSHelpers() bkQueryDict = bkQuery.getQueryDict() gLogger.notice("For BK Query:", str(bkQueryDict)) progressBar = ProgressBar(1, title="Running BK query...", step=1) res = bk.getFilesWithMetadata(bkQueryDict) if not res['OK']: gLogger.error("Error getting files from BK", res['Message']) DIRAC.exit(2) if 'ParameterNames' in res.get('Value', {}): parameterNames = res['Value']['ParameterNames'] info = res['Value']['Records'] progressBar.endLoop("Obtained %d files" % len(info)) else: gLogger.error('\nNo metadata found') DIRAC.exit(3) lfns = [] runLFNs = {} for item in info: metadata = dict(zip(parameterNames, item)) lfn = metadata['FileName'] lfns.append(lfn) runLFNs.setdefault(metadata['RunNumber'], []).append(lfn) chunkSize = 1000 progressBar = ProgressBar(len(lfns), title='Getting replicas of %d files' % len(lfns), chunk=chunkSize) replicas = {} errors = {} for lfnChunk in breakListIntoChunks(lfns, chunkSize): progressBar.loop() res = dm.getReplicas(lfnChunk, getUrl=False) if not res['OK']: errors.setdefault(res['Message'], []).extend(lfnChunk) else: replicas.update(res['Value']['Successful']) for lfn, error in res['Value']['Failed'].iteritems(): errors.setdefault(error, []).append(lfn) progressBar.endLoop() for error, lfns in errors.iteritems(): gLogger.error(error, 'for %d files' % len(lfns)) tier1RDST = set(resolveSEGroup('Tier1-RDST')) setOK = 0 errors = {} progressBar = ProgressBar(len(runLFNs), title='Defining destination for %d runs' % len(runLFNs), step=10) for run, lfns in runLFNs.iteritems(): progressBar.loop() res = tr.getDestinationForRun(run) if res.get('Value'): errors.setdefault('Destination already set', []).append(str(run)) continue # print 'Run', run, len( lfns ), 'Files', lfns[:3] seCounts = {} for lfn in lfns: for se in tier1RDST.intersection(replicas.get(lfn, [])): seCounts[se] = seCounts.setdefault(se, 0) + 1 # print seCounts maxi = 0 seMax = None for se, count in seCounts.iteritems(): if count > maxi: seMax = se maxi = count if not seMax: errors.setdefault('No SE found, use CERN-RDST', []).append(str(run)) seMax = 'CERN-RDST' # SE found, get its site res = dmsHelper.getLocalSiteForSE(seMax) if res['OK']: site = res['Value'] res = tr.setDestinationForRun(run, site) if not res['OK']: errors.setdefault(res['Message'], []).append(str(run)) else: setOK += 1 progressBar.endLoop('Successfully set destination for %d runs' % setOK) for error, runs in errors.iteritems(): gLogger.error(error, 'for runs %s' % ','.join(runs))