def __init__( self, catalogs = None, vo = None ): """ Default constructor """ self.valid = True self.timeout = 180 self.readCatalogs = [] self.writeCatalogs = [] self.metaCatalogs = [] self.rootConfigPath = '/Resources/FileCatalogs' self.vo = vo if vo else getVOfromProxyGroup().get( 'Value', None ) self.opHelper = Operations( vo = self.vo ) if catalogs is None: catalogList = [] elif type( catalogs ) in types.StringTypes: catalogList = [catalogs] else: catalogList = catalogs if catalogList: res = self._getSelectedCatalogs( catalogList ) else: res = self._getCatalogs() if not res['OK']: self.valid = False elif ( len( self.readCatalogs ) == 0 ) and ( len( self.writeCatalogs ) == 0 ): self.valid = False
def __init__( self, catalogs = [], vo = None ): """ Default constructor """ self.valid = True self.timeout = 180 self.readCatalogs = [] self.writeCatalogs = [] self.vo = vo if not vo: result = getVOfromProxyGroup() if not result['OK']: return result self.vo = result['Value'] self.opHelper = Operations( vo = self.vo ) self.reHelper = Resources( vo = self.vo ) if type( catalogs ) in types.StringTypes: catalogs = [catalogs] if catalogs: res = self._getSelectedCatalogs( catalogs ) else: res = self._getCatalogs() if not res['OK']: self.valid = False elif ( len( self.readCatalogs ) == 0 ) and ( len( self.writeCatalogs ) == 0 ): self.valid = False
def getCEStatus( self ): """ Method to return information on running and pending jobs. """ vo = '' res = getVOfromProxyGroup() if res['OK']: vo = res['Value'] else: # A backup solution which may work vo = self.ceParameters['VO'] cmd = 'ldapsearch -x -LLL -H ldap://%s:2135 -b mds-vo-name=resource,o=grid "(GlueVOViewLocalID=%s)"' %(self.ceHost, vo.lower()) res = shellCall( 0, cmd ) if not res['OK']: gLogger.debug("Could not query CE %s - is it down?" % self.ceHost) return res result = S_OK() try: ldapValues = res['Value'][1].split("\n") running = [y for y in ldapValues if 'GlueCEStateRunningJobs' in y] waiting = [y for y in ldapValues if 'GlueCEStateWaitingJobs' in y] result['RunningJobs'] = int(running[0].split(":")[1]) result['WaitingJobs'] = int(waiting[0].split(":")[1]) except IndexError: result = S_ERROR('Unknown ldap failure for site %s' % self.ceHost) result['RunningJobs'] = 0 result['WaitingJobs'] = 0 gLogger.debug("Running jobs for CE %s : %s" % (self.ceHost, result['RunningJobs'])) gLogger.debug("Waiting jobs for CE %s : %s" % (self.ceHost, result['WaitingJobs'])) result['SubmittedJobs'] = 0 return result
def __init__(self, catalogs=None, vo=None): """ Default constructor """ self.valid = True self.timeout = 180 self.readCatalogs = [] self.writeCatalogs = [] self.metaCatalogs = [] self.rootConfigPath = '/Resources/FileCatalogs' self.vo = vo if vo else getVOfromProxyGroup().get('Value', None) self.opHelper = Operations(vo=self.vo) if catalogs is None: catalogList = [] elif type(catalogs) in types.StringTypes: catalogList = [catalogs] else: catalogList = catalogs if catalogList: res = self._getSelectedCatalogs(catalogList) else: res = self._getCatalogs() if not res['OK']: self.valid = False elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0): self.valid = False
def __call__(self, name, plugins=None, vo=None, hideExceptions=False): self.seCache.purgeExpired(expiredInSeconds=60) tId = threading.current_thread().ident if not vo: result = getVOfromProxyGroup() if not result['OK']: return vo = result['Value'] # Because the gfal2 context caches the proxy location, # we also use the proxy location as a key. # In practice, there should almost always be one, except for the REA # If we see its memory consumtpion exploding, this might be a place to look proxyLoc = getProxyLocation() argTuple = (tId, name, plugins, vo, proxyLoc) seObj = self.seCache.get(argTuple) if not seObj: seObj = StorageElementItem(name, plugins, vo, hideExceptions=hideExceptions) # Add the StorageElement to the cache for 1/2 hour self.seCache.add(argTuple, 1800, seObj) return seObj
def getCEStatus(self): """ Method to return information on running and pending jobs. """ vo = '' res = getVOfromProxyGroup() if res['OK']: vo = res['Value'] else: # A backup solution which may work vo = self.ceParameters['VO'] cmd = 'ldapsearch -x -LLL -H ldap://%s:2135 -b mds-vo-name=resource,o=grid "(GlueVOViewLocalID=%s)"' % ( self.ceHost, vo.lower()) res = shellCall(0, cmd) if not res['OK']: gLogger.debug("Could not query CE %s - is it down?" % self.ceHost) return res result = S_OK() try: ldapValues = res['Value'][1].split("\n") running = [y for y in ldapValues if 'GlueCEStateRunningJobs' in y] waiting = [y for y in ldapValues if 'GlueCEStateWaitingJobs' in y] result['RunningJobs'] = int(running[0].split(":")[1]) result['WaitingJobs'] = int(waiting[0].split(":")[1]) except IndexError: result = S_ERROR('Unknown ldap failure for site %s' % self.ceHost) result['RunningJobs'] = 0 result['WaitingJobs'] = 0 gLogger.debug("Running jobs for CE %s : %s" % (self.ceHost, result['RunningJobs'])) gLogger.debug("Waiting jobs for CE %s : %s" % (self.ceHost, result['WaitingJobs'])) result['SubmittedJobs'] = 0 return result
def __init__(self, catalogs=[], vo=None): """ Default constructor """ self.valid = True self.timeout = 180 self.readCatalogs = [] self.writeCatalogs = [] self.vo = vo if not vo: result = getVOfromProxyGroup() if not result['OK']: return result self.vo = result['Value'] self.opHelper = Operations(vo=self.vo) self.reHelper = Resources(vo=self.vo) if type(catalogs) in types.StringTypes: catalogs = [catalogs] if catalogs: res = self._getSelectedCatalogs(catalogs) else: res = self._getCatalogs() if not res['OK']: self.valid = False elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0): self.valid = False
def getCEStatus(self): """ Method to return information on running and pending jobs. We hope to satisfy both instances that use robot proxies and those which use proper configurations. """ result = self._prepareProxy() self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"]) if not result["OK"]: gLogger.error("ARCComputingElement: failed to set up proxy", result["Message"]) return result # Try to find out which VO we are running for. vo = "" res = getVOfromProxyGroup() if res["OK"]: vo = res["Value"] result = S_OK() result["SubmittedJobs"] = 0 if not vo: # Presumably the really proper way forward once the infosys-discuss WG comes up with a solution # and it is implemented. Needed for DIRAC instances which use robot certificates for pilots. endpoints = [ arc.Endpoint( "ldap://" + self.ceHost + "/MDS-Vo-name=local,o=grid", arc.Endpoint.COMPUTINGINFO, "org.nordugrid.ldapng", ) ] retriever = arc.ComputingServiceRetriever(self.usercfg, endpoints) retriever.wait() # Takes a bit of time to get and parse the ldap information targets = retriever.GetExecutionTargets() ceStats = targets[0].ComputingShare gLogger.debug("Running jobs for CE %s : %s" % (self.ceHost, ceStats.RunningJobs)) gLogger.debug("Waiting jobs for CE %s : %s" % (self.ceHost, ceStats.WaitingJobs)) result["RunningJobs"] = ceStats.RunningJobs result["WaitingJobs"] = ceStats.WaitingJobs else: # The system which works properly at present for ARC CEs that are configured correctly. # But for this we need the VO to be known - ask me (Raja) for the whole story if interested. cmd = 'ldapsearch -x -LLL -H ldap://%s:2135 -b mds-vo-name=resource,o=grid "(GlueVOViewLocalID=%s)"' % ( self.ceHost, vo.lower(), ) res = shellCall(0, cmd) if not res["OK"]: gLogger.debug("Could not query CE %s - is it down?" % self.ceHost) return res try: ldapValues = res["Value"][1].split("\n") running = [lValue for lValue in ldapValues if "GlueCEStateRunningJobs" in lValue] waiting = [lValue for lValue in ldapValues if "GlueCEStateWaitingJobs" in lValue] result["RunningJobs"] = int(running[0].split(":")[1]) result["WaitingJobs"] = int(waiting[0].split(":")[1]) except IndexError: res = S_ERROR("Unknown ldap failure for site %s" % self.ceHost) return res return result
def createCatalog(self, catalogName, useProxy=False, vo=None, catalogConfig={}): """ Create a file catalog object from its name and CS description """ if useProxy: catalog = FileCatalogProxyClient(catalogName) return S_OK(catalog) # get the CS description first catConfig = catalogConfig if not catConfig: if not vo: result = getVOfromProxyGroup() if not result['OK']: return result vo = result['Value'] reHelper = Resources(vo=vo) result = reHelper.getCatalogOptionsDict(catalogName) if not result['OK']: return result catConfig = result['Value'] catalogType = catConfig.get('CatalogType', catalogName) catalogURL = catConfig.get('CatalogURL', '') self.log.verbose('Creating %s client' % catalogName) moduleRootPaths = getInstalledExtensions() for moduleRootPath in moduleRootPaths: gLogger.verbose("Trying to load from root path %s" % moduleRootPath) #moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogType ) #gLogger.verbose( "Looking for file %s" % moduleFile ) #if not os.path.isfile( moduleFile ): # continue try: # This enforces the convention that the plug in must be named after the file catalog moduleName = "%sClient" % (catalogType) catalogModule = __import__( '%s.Resources.Catalog.%s' % (moduleRootPath, moduleName), globals(), locals(), [moduleName]) except ImportError, x: if "No module" in str(x): gLogger.debug('Catalog module %s not found in %s' % (catalogType, moduleRootPath)) else: errStr = "Failed attempt to import %s from the path %s: %s" % ( catalogType, moduleRootPath, x) gLogger.error(errStr) continue except Exception, x: errStr = "Failed attempt to import %s from the path %s: %s" % ( catalogType, moduleRootPath, x) gLogger.error(errStr) continue
def createCatalog(self, catalogName, useProxy=False, vo=None, catalogConfig={}): """ Create a file catalog object from its name and CS description """ if useProxy: catalog = FileCatalogProxyClient(catalogName) return S_OK(catalog) # get the CS description first catConfig = catalogConfig if not catConfig: if not vo: result = getVOfromProxyGroup() if not result['OK']: return result vo = result['Value'] reHelper = Resources(vo=vo) result = reHelper.getCatalogOptionsDict(catalogName) if not result['OK']: return result catConfig = result['Value'] catalogType = catConfig.get('CatalogType', catalogName) catalogURL = catConfig.get('CatalogURL', '') self.log.verbose('Creating %s client' % catalogName) objectLoader = ObjectLoader() result = objectLoader.loadObject( 'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client') if not result['OK']: gLogger.error('Failed to load catalog object: %s' % result['Message']) return result catalogClass = result['Value'] try: if catalogType in ['LcgFileCatalogCombined', 'LcgFileCatalog']: # The LFC special case infoSys = catConfig.get('LcgGfalInfosys', '') host = catConfig.get('MasterHost', '') catalog = catalogClass(infoSys, host) else: if catalogURL: catalog = catalogClass(url=catalogURL) else: catalog = catalogClass() self.log.debug('Loaded module %sClient' % catalogType) return S_OK(catalog) except Exception, x: errStr = "Failed to instantiate %s()" % (catalogType) gLogger.exception(errStr, lException=x) return S_ERROR(errStr)
def main(): Script.setUsageMessage("\n".join([ "Get VM nodes information", "Usage:", "%s [option]... [cfgfile]" % Script.scriptName, "Arguments:", " cfgfile: DIRAC Cfg with description of the configuration (optional)", ])) Script.registerSwitch("S:", "Site=", "Site Name", setSite) Script.registerSwitch("C:", "CE=", "Cloud Endpoint Name ", setCE) Script.registerSwitch("I:", "Image=", "Image Name", setImage) Script.registerSwitch("v:", "vo=", "VO name", setVO) Script.parseCommandLine(ignoreErrors=True) args = Script.getExtraCLICFGFiles() from DIRAC.WorkloadManagementSystem.Client.VMClient import VMClient from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.Core.Utilities.PrettyPrint import printTable siteList = None if site is not None: siteList = [s.strip() for s in site.split(",")] ceList = None if ce is not None: ceList = [c.strip() for c in ce.split(",")] voName = vo if voName is None: result = getVOfromProxyGroup() if result["OK"]: voName = result["Value"] records = [] vmClient = VMClient() result = vmClient.getCEInstances(siteList, ceList, voName) if not result["OK"]: gLogger.error(result["Message"]) DIRACExit(-1) for nodeID in result["Value"]: nodeDict = result["Value"][nodeID] record = [ nodeDict["Site"], nodeDict["CEName"], nodeID, nodeDict["NodeName"], nodeDict["PublicIP"], nodeDict["State"], ] records.append(record) fields = ["Site", "Endpoint", "ID", "Name", "PublicIP", "State"] printTable(fields, records) DIRACExit(0)
def constructUserLFNs(jobID, vo, owner, outputFiles, outputPath): """ This method is used to supplant the standard job wrapper output data policy for ILC. The initial convention adopted for user output files is the following: If outputpath is not defined: <vo>/user/<initial e.g. s>/<owner e.g. sposs>/<yearMonth e.g. 2010_02>/<subdir>/<fileName> Otherwise: <vo>/user/<initial e.g. s>/<owner e.g. sposs>/<outputPath>/<fileName> """ initial = owner[:1] subdir = str(jobID/1000) timeTup = datetime.date.today().timetuple() yearMonth = '%s_%s' % (timeTup[0], string.zfill(str(timeTup[1]), 2)) outputLFNs = {} if not vo: #res = gConfig.getOption("/DIRAC/VirtualOrganization", "ilc") res = getVOfromProxyGroup() if not res['OK']: gLogger.error('Could not get VO from CS, assuming ilc') vo = 'ilc' else: vo = res['Value'] ops = Operations(vo = vo) lfn_prefix = ops.getValue("LFNUserPrefix", "user") #Strip out any leading or trailing slashes but allow fine structure if outputPath: outputPathList = string.split(outputPath, os.sep) newPath = [] for i in outputPathList: if i: newPath.append(i) outputPath = string.join(newPath, os.sep) if not type(outputFiles) == types.ListType: outputFiles = [outputFiles] for outputFile in outputFiles: #strip out any fine structure in the output file specified by the user, restrict to output file names #the output path field can be used to describe this outputFile = outputFile.replace('LFN:', '') lfn = '' if outputPath: lfn = os.sep+os.path.join(vo, lfn_prefix, initial, owner, outputPath + os.sep + os.path.basename(outputFile)) else: lfn = os.sep+os.path.join(vo, lfn_prefix, initial, owner, yearMonth, subdir, str(jobID)) + os.sep + os.path.basename(outputFile) outputLFNs[outputFile] = lfn outputData = outputLFNs.values() if outputData: gLogger.info('Created the following output data LFN(s):\n%s' % (string.join(outputData, '\n'))) else: gLogger.info('No output LFN(s) constructed') return S_OK(outputData)
def _getCatalogs( self ): # Get the eligible catalogs first # First, look in the Operations, if nothing defined look in /Resources for backward compatibility result = getVOfromProxyGroup() if not result['OK']: return result vo = result['Value'] opHelper = Operations( vo = vo ) result = opHelper.getSections( '/Services/FileCatalogs' ) fileCatalogs = [] operationsFlag = False if result['OK']: fileCatalogs = result['Value'] operationsFlag = True else: res = gConfig.getSections( self.rootConfigPath, listOrdered = True ) if not res['OK']: errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration." gLogger.error( errStr, res['Message'] ) return S_ERROR( errStr ) fileCatalogs = res['Value'] # Get the catalogs now for catalogName in fileCatalogs: res = self._getCatalogConfigDetails( catalogName ) if not res['OK']: return res catalogConfig = res['Value'] if operationsFlag: result = opHelper.getOptionsDict( '/Services/FileCatalogs/%s' % catalogName ) if not result['OK']: return result catalogConfig.update( result['Value'] ) if catalogConfig['Status'] == 'Active': res = self._generateCatalogObject( catalogName ) if not res['OK']: return res oCatalog = res['Value'] master = catalogConfig['Master'] # If the catalog is read type if re.search( 'Read', catalogConfig['AccessType'] ): if master: self.readCatalogs.insert( 0, ( catalogName, oCatalog, master ) ) else: self.readCatalogs.append( ( catalogName, oCatalog, master ) ) # If the catalog is write type if re.search( 'Write', catalogConfig['AccessType'] ): if master: self.writeCatalogs.insert( 0, ( catalogName, oCatalog, master ) ) else: self.writeCatalogs.append( ( catalogName, oCatalog, master ) ) return S_OK()
def getCEStatus(self): """ Method to return information on running and pending jobs. We hope to satisfy both instances that use robot proxies and those which use proper configurations. """ result = self._prepareProxy() if not result['OK']: gLogger.error('ARCComputingElement: failed to set up proxy', result['Message']) return result self.usercfg.ProxyPath(os.environ['X509_USER_PROXY']) # Try to find out which VO we are running for. vo = '' res = getVOfromProxyGroup() if res['OK']: vo = res['Value'] result = S_OK() result['SubmittedJobs'] = 0 if not vo: # Presumably the really proper way forward once the infosys-discuss WG comes up with a solution # and it is implemented. Needed for DIRAC instances which use robot certificates for pilots. endpoints = [arc.Endpoint("ldap://" + self.ceHost + "/MDS-Vo-name=local,o=grid", arc.Endpoint.COMPUTINGINFO, 'org.nordugrid.ldapng')] retriever = arc.ComputingServiceRetriever(self.usercfg, endpoints) retriever.wait() # Takes a bit of time to get and parse the ldap information targets = retriever.GetExecutionTargets() ceStats = targets[0].ComputingShare gLogger.debug("Running jobs for CE %s : %s" % (self.ceHost, ceStats.RunningJobs)) gLogger.debug("Waiting jobs for CE %s : %s" % (self.ceHost, ceStats.WaitingJobs)) result['RunningJobs'] = ceStats.RunningJobs result['WaitingJobs'] = ceStats.WaitingJobs else: # The system which works properly at present for ARC CEs that are configured correctly. # But for this we need the VO to be known - ask me (Raja) for the whole story if interested. cmd = 'ldapsearch -x -LLL -H ldap://%s:2135 -b mds-vo-name=resource,o=grid "(GlueVOViewLocalID=%s)"' % ( self.ceHost, vo.lower()) res = shellCall(0, cmd) if not res['OK']: gLogger.debug("Could not query CE %s - is it down?" % self.ceHost) return res try: ldapValues = res['Value'][1].split("\n") running = [lValue for lValue in ldapValues if 'GlueCEStateRunningJobs' in lValue] waiting = [lValue for lValue in ldapValues if 'GlueCEStateWaitingJobs' in lValue] result['RunningJobs'] = int(running[0].split(":")[1]) result['WaitingJobs'] = int(waiting[0].split(":")[1]) except IndexError: res = S_ERROR('Unknown ldap failure for site %s' % self.ceHost) return res return result
def createCatalog( self, catalogName, useProxy = False, vo = None, catalogConfig = {} ): """ Create a file catalog object from its name and CS description """ if useProxy: catalog = FileCatalogProxyClient( catalogName ) return S_OK( catalog ) # get the CS description first catConfig = catalogConfig if not catConfig: if not vo: result = getVOfromProxyGroup() if not result['OK']: return result vo = result['Value'] reHelper = Resources( vo = vo ) result = reHelper.getCatalogOptionsDict( catalogName ) if not result['OK']: return result catConfig = result['Value'] catalogType = catConfig.get('CatalogType',catalogName) catalogURL = catConfig.get('CatalogURL','') self.log.verbose( 'Creating %s client' % catalogName ) objectLoader = ObjectLoader() result = objectLoader.loadObject( 'Resources.Catalog.%sClient' % catalogType, catalogType+'Client' ) if not result['OK']: gLogger.error( 'Failed to load catalog object: %s' % result['Message'] ) return result catalogClass = result['Value'] try: if catalogType in ['LcgFileCatalogCombined','LcgFileCatalog']: # The LFC special case infoSys = catConfig.get('LcgGfalInfosys','') host = catConfig.get('MasterHost','') catalog = catalogClass( infoSys, host ) else: if catalogURL: catalog = catalogClass( url = catalogURL ) else: catalog = catalogClass() self.log.debug('Loaded module %sClient' % catalogType ) return S_OK( catalog ) except Exception, x: errStr = "Failed to instantiate %s()" % ( catalogType ) gLogger.exception( errStr, lException = x ) return S_ERROR( errStr )
def __init__(self, useProxy=False, vo=None): self.proxy = False self.proxy = useProxy self.resourceStatus = ResourceStatus() self.vo = vo if self.vo is None: result = getVOfromProxyGroup() if result['OK']: self.vo = result['Value'] else: RuntimeError("Can not get the current VO context") self.remotePlugins = [] self.localPlugins = [] self.name = '' self.options = {} self.protocols = {} self.storages = []
def __init__( self, useProxy = False, vo = None ): self.rootConfigPath = '/Resources/StorageElements' self.proxy = False self.proxy = useProxy self.resourceStatus = ResourceStatus() self.vo = vo if self.vo is None: result = getVOfromProxyGroup() if result['OK']: self.vo = result['Value'] else: RuntimeError( "Can not get the current VO context" ) self.remotePlugins = [] self.localPlugins = [] self.name = '' self.options = {} self.protocolDetails = [] self.storages = []
def __call__( self, name, plugins = None, vo = None, hideExceptions = False ): self.seCache.purgeExpired( expiredInSeconds = 60 ) tId = threading.current_thread().ident if not vo: result = getVOfromProxyGroup() if not result['OK']: return vo = result['Value'] argTuple = ( tId, name, plugins, vo ) seObj = self.seCache.get( argTuple ) if not seObj: seObj = StorageElementItem( name, plugins, vo, hideExceptions = hideExceptions ) # Add the StorageElement to the cache for 1/2 hour self.seCache.add( argTuple, 1800, seObj ) return seObj
def __discoverSettings( self ): #Set the VO globalVO = CSGlobals.getVO() if globalVO: self.__vo = globalVO elif self.__uVO: self.__vo = self.__uVO elif self.__uGroup: self.__vo = Registry.getVOForGroup( self.__uGroup ) if not self.__vo: self.__vo = False else: result = getVOfromProxyGroup() if result['OK']: self.__vo = result['Value'] #Set the setup self.__setup = False if self.__uSetup: self.__setup = self.__uSetup else: self.__setup = CSGlobals.getSetup()
def __discoverSettings(self): #Set the VO globalVO = CSGlobals.getVO() if globalVO: self.__vo = globalVO elif self.__uVO: self.__vo = self.__uVO elif self.__uGroup: self.__vo = Registry.getVOForGroup(self.__uGroup) if not self.__vo: self.__vo = False else: result = getVOfromProxyGroup() if result['OK']: self.__vo = result['Value'] #Set the setup self.__setup = False if self.__uSetup: self.__setup = self.__uSetup else: self.__setup = CSGlobals.getSetup()
def __discoverSettings(self): """Discovers the vo and the setup""" # Set the VO globalVO = CSGlobals.getVO() if globalVO: self.__vo = globalVO elif self.__uVO: self.__vo = self.__uVO elif self.__uGroup: self.__vo = Registry.getVOForGroup(self.__uGroup) if not self.__vo: self.__vo = False else: result = getVOfromProxyGroup() if result["OK"]: self.__vo = result["Value"] # Set the setup self.__setup = False if self.__uSetup: self.__setup = self.__uSetup else: self.__setup = CSGlobals.getSetup()
def getCEStatus( self ): """ Method to return information on running and pending jobs. """ vo = '' result = getVOfromProxyGroup() if result['OK']: vo = result['Value'] else: # A backup solution which may work vo = self.ceParameters['VO'] voFilters = '(GlueCEAccessControlBaseRule=VOMS:/%s/*)' % vo voFilters += '(GlueCEAccessControlBaseRule=VOMS:/%s)' % vo voFilters += '(GlueCEAccessControlBaseRule=VO:%s)' % vo filt = '(&(GlueCEUniqueID=%s*)(|%s))' % ( self.ceHost, voFilters ) result = ldapsearchBDII( filt, attr=None, host=None, base=None ) ces = result['Value'] filt = '(&(objectClass=GlueVOView)(|%s))' % ( voFilters ) dn = ces[0]['dn'] result = ldapsearchBDII( filt, attr=None, host=None, base = dn ) stats = result['Value'][0]['attr'] result['RunningJobs'] = int(stats["GlueCEStateRunningJobs"]) result['WaitingJobs'] = int(stats["GlueCEStateTotalJobs"]) result['SubmittedJobs'] = 0 return result
def getReplicas(self, lfns, allStatus=False, timeout=120): """ Get the replicas of the given files """ rpcClient = self._getRPC(timeout=timeout) result = rpcClient.getReplicas(lfns, allStatus) if not result['OK']: return result vo = getVOfromProxyGroup().get('Value', None) lfnDict = result['Value'] seDict = result['Value'].get('SEPrefixes', {}) for lfn in lfnDict['Successful']: for se in lfnDict['Successful'][lfn]: if not lfnDict['Successful'][lfn][se]: # The PFN was not returned, construct it on the fly # For some VO's the prefix can be non-standard voPrefix = seDict.get("VOPrefix", {}).get(se, {}).get(vo) sePrefix = seDict.get(se, '') prefix = voPrefix if voPrefix else sePrefix lfnDict['Successful'][lfn][se] = prefix + lfn return S_OK(lfnDict)
ces_current_site = sites_ce[site] for ce in ces_current_site: if ce in vo_ces: curr_ces = final_dict[site]["CE"] curr_ces.append(ce) final_dict[site].update({"Tags":ldapTag(ce,vo),"CE":curr_ces}) #final_dict[site]={"Tags":ldapTag(ce,vo),"CE":[ce]} ret_dict = {} for key in final_dict: if len(final_dict[key]['CE'])!=0: ret_dict[key]=final_dict[key] return ret_dict if __name__ == "__main__": Script.parseCommandLine() vo = "glast.org" res = getVOfromProxyGroup() if not res['OK']: gLogger.error(res['Message']) gLogger.error('Could not get VO from CS, assuming glast.org') dexit(1) else: gLogger.info(res) vo = res['Value'] d = main(vo) for key in d: print('Name of DIRAC site %s\nName of CEs: %s'%(key,str(d[key]["CE"]))) for tag in d[key]["Tags"]: print('\t%s'%tag) print('\n')
def main(): from DIRAC import S_OK, gLogger, gConfig, exit as DIRACExit ceFlag = False seFlag = False voName = None def setCEFlag(args_): global ceFlag ceFlag = True def setSEFlag(args_): global seFlag seFlag = True def setVOName(args): global voName voName = args Script.registerSwitch("C", "ce", "Get CE info", setCEFlag) Script.registerSwitch("S", "se", "Get SE info", setSEFlag) Script.registerSwitch( "V:", "vo=", "Get resources for the given VO. If not set, taken from the proxy", setVOName) Script.parseCommandLine(ignoreErrors=True) from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.ConfigurationSystem.Client.Helpers import Resources from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus def printCEInfo(voName): resultQueues = Resources.getQueues(community=voName) if not resultQueues["OK"]: gLogger.error("Failed to get CE information") DIRACExit(-1) fields = ("Site", "CE", "CEType", "Queue", "Status") records = [] # get list of usable sites within this cycle resultMask = SiteStatus().getUsableSites() if not resultMask["OK"]: return resultMask siteMaskList = resultMask.get("Value", []) rssClient = ResourceStatus() for site in resultQueues["Value"]: siteStatus = "Active" if site in siteMaskList else "InActive" siteNew = True for ce in resultQueues["Value"][site]: ceStatus = siteStatus if rssClient.rssFlag: result = rssClient.getElementStatus(ce, "ComputingElement") if result["OK"]: ceStatus = result["Value"][ce]["all"] ceNew = True for queue in resultQueues["Value"][site][ce]["Queues"]: pSite = site if siteNew else "" pCE = "" ceType = "" if ceNew: pCE = ce ceType = resultQueues["Value"][site][ce]["CEType"] records.append((pSite, pCE, ceType, queue, ceStatus)) ceNew = False siteNew = False gLogger.notice( printTable(fields, records, printOut=False, columnSeparator=" ")) return S_OK() def printSEInfo(voName): fields = ("SE", "Status", "Protocols", "Aliases") records = [] for se in DMSHelpers(voName).getStorageElements( ): # this will get the full list of SEs, not only the vo's ones. seObject = StorageElement(se) if not (seObject.vo and voName in seObject.vo.strip().split(",") or not seObject.voName): continue result = seObject.status() status = [] for statusType in ["Write", "Read"]: if result[statusType]: status.append(statusType) if status: status = "/".join(status) else: status = "InActive" records.append((se, status, ",".join([ seProtocol["Protocol"] for seProtocol in seObject.protocolOptions ]))) gLogger.notice( printTable(fields, records, printOut=False, columnSeparator=" ")) return S_OK() if not voName: # Get the current VO result = getVOfromProxyGroup() if not result["OK"]: gLogger.error("No proxy found, please login") DIRACExit(-1) voName = result["Value"] else: result = gConfig.getSections("/Registry/VO") if not result["OK"]: gLogger.error("Failed to contact the CS") DIRACExit(-1) if voName not in result["Value"]: gLogger.error("Invalid VO name") DIRACExit(-1) if not (ceFlag or seFlag): gLogger.error("Resource type is not specified") DIRACExit(-1) if ceFlag: result = printCEInfo(voName) if not result["OK"]: gLogger.error(result["Message"]) DIRACExit(-1) if seFlag: result = printSEInfo(voName) if not result["OK"]: gLogger.error(result["Message"]) DIRACExit(-1) DIRACExit(0)
def constructProductionLFNs(paramDict): """ Used for local testing of a workflow, a temporary measure until LFN construction is tidied. This works using the workflow commons for on the fly construction. """ keys = ['PRODUCTION_ID', 'JOB_ID', 'JobType', 'outputList'] for k in keys: if not paramDict.has_key(k): return S_ERROR('%s not defined' % k) productionID = paramDict['PRODUCTION_ID'] jobID = paramDict['JOB_ID'] # wfMode = paramDict['dataType'] #wfLfnprefix=paramDict['lfnprefix'] #wfLfnpostfix=paramDict['lfnpostfix'] wfMask = "" # wfMask = paramDict['outputDataFileMask'] if not type(wfMask) == type([]): wfMask = [i.lower().strip() for i in wfMask.split(';')] wfType = paramDict['JobType'] outputList = paramDict['outputList'] inputData = '' if paramDict.has_key('InputData'): inputData = paramDict['InputData'] res = getVOfromProxyGroup() #res = gConfig.getOption("/DIRAC/VirtualOrganization", "ilc") if not res['OK']: gLogger.error('Could not get VO from CS, assuming ilc') vo = 'ilc' else: vo = res['Value'] fileTupleList = [] #gLogger.verbose('wfLfnprefix = %s, wfLfnpostfix = %s, wfMask = %s, wfType=%s' %(wfLfnprefix,wfLfnpostfix,wfMask,wfType)) gLogger.verbose('outputList %s' % (outputList)) for info in outputList: #Nasty check on whether the created code parameters were not updated e.g. when changing defaults in a workflow fileName = info['outputFile'] #rename to take care of correct path fileName = getProdFilename(fileName, int(productionID), int(jobID)) #index=0 #if not re.search('^\d',fileName[index]): # index+=1 #if not fileName[index]==str(productionID).zfill(8): # fileName[index]=str(productionID).zfill(8) #if not fileName[index+1]==str(jobID).zfill(8): # fileName[index+1]=str(jobID).zfill(8) fileTupleList.append((info['outputPath'], fileName)) lfnRoot = '' debugRoot = '' #if inputData: # gLogger.verbose('Making LFN_ROOT for job with inputdata: %s' %(inputData)) # lfnRoot = _getLFNRoot(inputData,wfLfnpostfix) # debugRoot= _getLFNRoot('','debug',wfLfnpostfix) #else: # lfnRoot = _getLFNRoot('',wfLfnprefix,wfLfnpostfix) # gLogger.verbose('LFN_ROOT is: %s' %(lfnRoot)) # debugRoot= _getLFNRoot('','debug',wfLfnpostfix) #lfnRoot = #gLogger.verbose('LFN_ROOT is: %s' %(lfnRoot)) #if not lfnRoot: # return S_ERROR('LFN root could not be constructed') #Get all LFN(s) to both output data and BK lists at this point (fine for BK) outputData = [] #bkLFNs = [] debugLFNs = [] for fileTuple in fileTupleList: #lfn = _makeProductionLfn(str(jobID).zfill(8),lfnRoot,fileTuple,wfLfnprefix,str(productionID).zfill(8)) lfn = fileTuple[0] + "/" + str(productionID).zfill(8) + "/" + str(int(jobID)/1000).zfill(3) + "/" + fileTuple[1] if lfn.count('//'): lfn = lfn.replace('//','/') outputData.append(lfn) #bkLFNs.append(lfn) if debugRoot: #debugLFNs.append(_makeProductionLfn(str(jobID).zfill(8),debugRoot,fileTuple,wfLfnprefix,str(productionID).zfill(8))) debugLFNs.append("/" + vo + "/prod/debug/" + str(productionID).zfill(8)) #if debugRoot: # debugLFNs.append(_makeProductionLfn(str(jobID).zfill(8),debugRoot,('%s_core' % str(jobID).zfill(8) ,'core'),wfLfnprefix,str(productionID).zfill(8))) #Get log file path - unique for all modules #logPath = _makeProductionPath(str(jobID).zfill(8),lfnRoot,'LOG',wfLfnprefix,str(productionID).zfill(8),log=True) logPathtemp = fileTupleList[0][0].split("/") logPathroot = string.join(logPathtemp[0:len(logPathtemp)-1], "/") #TODO adjust for ILD logPath = logPathroot + "/LOG/" + str(productionID).zfill(8) logFilePath = ['%s/%s' % (logPath, str(int(jobID)/1000).zfill(3))] logTargetPath = ['%s/%s_%s.tar' % (logPath, str(productionID).zfill(8), str(int(jobID)).zfill(3))] #[ aside, why does makeProductionPath not append the jobID itself ???? # this is really only used in one place since the logTargetPath is just written to a text file (should be reviewed)... ] #Strip output data according to file mask if wfMask: newOutputData = [] #newBKLFNs = [] for od in outputData: for i in wfMask: if re.search('.%s$' % i, od): if not od in newOutputData: newOutputData.append(od) #for bk in bkLFNs: # newBKLFNs.append(bk) outputData = newOutputData #bkLFNs = newBKLFNs if not outputData: gLogger.info('No output data LFN(s) constructed') else: gLogger.verbose('Created the following output data LFN(s):\n%s' % (string.join(outputData,'\n'))) gLogger.verbose('Log file path is:\n%s' % logFilePath[0]) gLogger.verbose('Log target path is:\n%s' % logTargetPath[0]) #if bkLFNs: # gLogger.verbose('BookkeepingLFN(s) are:\n%s' %(string.join(bkLFNs,'\n'))) if debugLFNs: gLogger.verbose('DebugLFN(s) are:\n%s' % (string.join(debugLFNs, '\n'))) jobOutputs = {'ProductionOutputData' : outputData, 'LogFilePath' : logFilePath, 'LogTargetPath' : logTargetPath, 'DebugLFNs' : debugLFNs} return S_OK(jobOutputs)
def main(): Script.registerSwitch("", "FromSE=", "SE1[,SE2,...]") Script.registerSwitch("", "TargetSE=", "SE1[,SE2,...]") Script.registerSwitch( "", "OutputFile=", "CSV output file (default /tmp/protocol-matrix.csv)") Script.registerSwitch( "", "Bidirection", "If FromSE or TargetSE are specified, make a square matrix ") Script.registerSwitch("", "FTS", "Display the protocols sent to FTS") Script.registerSwitch("", "TPC", "Display the protocols tried for interactive TPC") Script.registerSwitch("", "Multihop", "Display the intermediate hop") Script.registerSwitch("", "Full", "Do not factorize with base SE") Script.registerSwitch("", "ExcludeSE=", "SEs to not take into account for the matrix") Script.parseCommandLine() from DIRAC import gConfig, gLogger, S_ERROR from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers from DIRAC.DataManagementSystem.private.FTS3Utilities import getFTS3Plugin from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup fromSE = [] targetSE = [] excludeSE = [] outputFile = "/tmp/protocol-matrix.csv" bidirection = False ftsTab = False tpcTab = False multihopTab = False fullOutput = False for switch in Script.getUnprocessedSwitches(): if switch[0] == "FromSE": fromSE = switch[1].split(",") elif switch[0] == "TargetSE": targetSE = switch[1].split(",") elif switch[0] == "ExcludeSE": excludeSE = switch[1].split(",") elif switch[0] == "OutputFile": outputFile = switch[1] elif switch[0] == "Bidirection": bidirection = True elif switch[0] == "FTS": ftsTab = True elif switch[0] == "TPC": tpcTab = True elif switch[0] == "Multihop": multihopTab = True elif switch[0] == "Full": fullOutput = True if not any([ftsTab, tpcTab, multihopTab]): ftsTab = tpcTab = multihopTab = True fts3Plugin = getFTS3Plugin() thirdPartyProtocols = DMSHelpers().getThirdPartyProtocols() # List all the BaseSE seBases = gConfig.getSections("/Resources/StorageElementBases")["Value"] # construct a dict { baseSE : <inherited storages>} seForSeBases = {} allSEs = gConfig.getSections("/Resources/StorageElements/")["Value"] # Remove the SEs that we want to exclude allSEs = set(allSEs) - set(excludeSE) # We go through all the SEs and fill in the seForSEBases dict. # Basically, at the end of the loop, the dict will contain # for each baseSE an entry corresponding to one real storage (the first one) # and itself for each real non inherited SE for se in allSEs: baseSE = gConfig.getOption("/Resources/StorageElements/%s/BaseSE" % se).get("Value") if baseSE and not fullOutput: if baseSE not in seForSeBases: seForSeBases[baseSE] = se else: # If no baseSE, we put self seForSeBases[se] = se # Now let's take into account what source and destination we want. # If the user did not specify source or dest, generate everything if not fromSE and not targetSE: fromSE = list(seForSeBases) targetSE = list(seForSeBases) else: # he specified at least source of dest # if bidirection, source and target should be the same if bidirection: if not fromSE and targetSE: # we gave target, but no source fromSE = targetSE elif fromSE and not targetSE: # we gave source but no target targetSE = fromSE elif fromSE and targetSE: # we gave both fromSE = targetSE = list(set(fromSE + targetSE)) else: # no bidirection # he specified a targetSE if not fromSE: fromSE = list(seForSeBases) elif not targetSE: targetSE = list(seForSeBases) fromSE = sorted(fromSE) targetSE = sorted(targetSE) gLogger.notice("Using sources: %s" % ",".join(fromSE)) gLogger.notice("Using target: %s" % ",".join(targetSE)) # Now we construct the SE object for each SE that we want to appear ses = {} for se in set(fromSE + targetSE): ses[se] = StorageElement(seForSeBases.get(se, se)) ret = getVOfromProxyGroup() if not ret["OK"] or not ret.get("Value", ""): gLogger.error("Aborting, Bad Proxy:", ret.get("Message", "Proxy does not belong to a VO!")) exit(1) vo = ret["Value"] gLogger.notice("Using the Virtual Organization:", vo) # dummy LFN, still has to follow lfn convention lfn = "/%s/toto.xml" % vo # Create a matrix of protocol src/dest tpMatrix = defaultdict(dict) ftsMatrix = defaultdict(dict) multihopMatrix = defaultdict(dict) # For each source and destination, generate the url pair, and the compatible third party protocols for src, dst in ((x, y) for x in fromSE for y in targetSE): if ftsTab: try: # breakpoint() fts3TpcProto = fts3Plugin.selectTPCProtocols( sourceSEName=ses[src].name, destSEName=ses[dst].name) res = ses[dst].generateTransferURLsBetweenSEs( lfn, ses[src], fts3TpcProto) except ValueError as e: res = S_ERROR(str(e)) if not res["OK"]: surls = "Error" gLogger.notice( "Could not generate transfer URLS", "src:%s, dst:%s, error:%s" % (src, dst, res["Message"])) else: # We only keep the protocol part of the url surls = "/".join(res["Value"]["Protocols"]) ftsMatrix[src][dst] = "%s" % surls gLogger.verbose("%s -> %s: %s" % (src, dst, surls)) # Add also the third party protocols if tpcTab: proto = ",".join(ses[dst].negociateProtocolWithOtherSE( ses[src], thirdPartyProtocols)["Value"]) tpMatrix[src][dst] = "%s" % proto gLogger.verbose("%s -> %s: %s" % (src, dst, proto)) if multihopTab: hop = fts3Plugin.findMultiHopSEToCoverUpForWLCGFailure( ses[src].name, ses[dst].name) multihopMatrix[src][dst] = hop # Write the matrix in the file with open(outputFile, "w") as csvfile: csvWriter = csv.writer(csvfile, delimiter=";", quoting=csv.QUOTE_MINIMAL) if tpcTab: csvWriter.writerow(["Direct TPC"]) csvWriter.writerow(["src/dst"] + targetSE) for src in fromSE: srcRow = [src] for dst in targetSE: srcRow.append(tpMatrix[src].get(dst, "NA")) csvWriter.writerow(srcRow) # make an empty line # csvWriter.writerow([""] * (len(targetSE) + 1)) csvWriter.writerow([]) csvWriter.writerow([]) csvWriter.writerow([]) if ftsTab: csvWriter.writerow(["FTS3 transfers"]) csvWriter.writerow(["src/dst"] + targetSE) for src in fromSE: srcRow = [src] for dst in targetSE: srcRow.append(ftsMatrix[src].get(dst, "NA")) csvWriter.writerow(srcRow) csvWriter.writerow([]) csvWriter.writerow([]) csvWriter.writerow([]) if multihopTab: csvWriter.writerow(["Multihop"]) csvWriter.writerow(["src/dst"] + targetSE) for src in fromSE: srcRow = [src] for dst in targetSE: srcRow.append(multihopMatrix[src].get(dst, "NA")) csvWriter.writerow(srcRow) csvWriter.writerow([]) csvWriter.writerow([]) csvWriter.writerow([]) gLogger.notice("Wrote Matrix to", outputFile)
def main(): global vo global noVOFlag global allVOsFlag Script.registerSwitch("V:", "vo=", "Virtual Organization", setVO) Script.registerSwitch("a", "all", "All Virtual Organizations flag", setAllVO) Script.registerSwitch("n", "noVO", "No Virtual Organizations assigned flag", setNoVO) Script.parseCommandLine() from DIRAC import gConfig, gLogger from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup storageCFGBase = "/Resources/StorageElements" res = gConfig.getSections(storageCFGBase, True) if not res['OK']: gLogger.error('Failed to get storage element info') gLogger.error(res['Message']) DIRACexit(1) gLogger.info("%s %s %s" % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15))) seList = sorted(res['Value']) resourceStatus = ResourceStatus() res = resourceStatus.getElementStatus(seList, "StorageElement") if not res['OK']: gLogger.error("Failed to get StorageElement status for %s" % str(seList)) DIRACexit(1) fields = ['SE', 'ReadAccess', 'WriteAccess', 'RemoveAccess', 'CheckAccess'] records = [] if vo is None and not allVOsFlag: result = getVOfromProxyGroup() if not result['OK']: gLogger.error('Failed to determine the user VO') DIRACexit(1) vo = result['Value'] for se, statusDict in res['Value'].items(): # Check if the SE is allowed for the user VO if not allVOsFlag: voList = gConfig.getValue('/Resources/StorageElements/%s/VO' % se, []) if noVOFlag and voList: continue if voList and vo not in voList: continue record = [se] for status in fields[1:]: value = statusDict.get(status, 'Unknown') record.append(value) records.append(record) printTable(fields, records, numbering=False, sortField='SE') DIRACexit(0)
gLogger.info( "%s %s %s" % ( 'Storage Element'.ljust( 25 ), 'Read Status'.rjust( 15 ), 'Write Status'.rjust( 15 ) ) ) seList = sorted( res[ 'Value' ] ) resourceStatus = ResourceStatus() res = resourceStatus.getStorageElementStatus( seList ) if not res[ 'OK' ]: gLogger.error( "Failed to get StorageElement status for %s" % str( seList ) ) DIRAC.exit( 1 ) fields = ['SE','ReadAccess','WriteAccess','RemoveAccess','CheckAccess'] records = [] result = getVOfromProxyGroup() if not result['OK']: gLogger.error( 'Failed to determine the user VO' ) DIRAC.exit( -1 ) vo = result['Value'] for se, statusDict in res[ 'Value' ].items(): # Check if the SE is allowed for the user VO voList = gConfig.getValue( '/Resources/StorageElements/%s/VO' % se, [] ) if voList and not vo in voList: continue record = [se] for status in fields[1:]: value = statusDict.get( status, 'Unknown' )
def constructUserLFNs(jobID, vo, owner, outputFiles, outputPath): """ This method is used to supplant the standard job wrapper output data policy for ILC. The initial convention adopted for user output files is the following: If outputpath is not defined: * <vo>/user/<initial e.g. s>/<owner e.g. sposs>/<yearMonth e.g. 2010_02>/<subdir>/<fileName> Otherwise: * <vo>/user/<initial e.g. s>/<owner e.g. sposs>/<outputPath>/<fileName> :param int jobID: the jobID :param string vo: the vo of the owners proxy :param string owner: the username :param list outputFiles: the list of outputfiles found for the job :param string outputPath: the outputpath defined for the job :returns: S_OK with list of output file lfns """ initial = owner[:1] subdir = str(jobID / 1000) timeTup = datetime.date.today().timetuple() yearMonth = '%s_%s' % (timeTup[0], string.zfill(str(timeTup[1]), 2)) outputLFNs = {} if not vo: res = getVOfromProxyGroup() if not res['OK']: gLogger.error('Could not get VO from CS, assuming ilc') vo = 'ilc' else: vo = res['Value'] ops = Operations(vo=vo) lfn_prefix = ops.getValue("LFNUserPrefix", "user") #Strip out any leading or trailing slashes but allow fine structure if outputPath: outputPathList = string.split(outputPath, os.sep) newPath = [] for i in outputPathList: if i: newPath.append(i) outputPath = string.join(newPath, os.sep) if not isinstance(outputFiles, list): outputFiles = [outputFiles] for outputFile in outputFiles: #strip out any fine structure in the output file specified by the user, restrict to output file names #the output path field can be used to describe this outputFile = outputFile.replace('LFN:', '') lfn = '' if outputPath: lfn = os.sep + os.path.join( vo, lfn_prefix, initial, owner, outputPath + os.sep + os.path.basename(outputFile)) else: lfn = os.sep + os.path.join( vo, lfn_prefix, initial, owner, yearMonth, subdir, str(jobID)) + os.sep + os.path.basename(outputFile) outputLFNs[outputFile] = lfn outputData = outputLFNs.values() if outputData: gLogger.info('Created the following output data LFN(s):\n%s' % (string.join(outputData, '\n'))) else: gLogger.info('No output LFN(s) constructed') return S_OK(outputData)
def __init__(self, name, protocols=None, vo=None): """ c'tor :param str name: SE name :param list protocols: requested protocols """ self.vo = vo if not vo: result = getVOfromProxyGroup() if not result["OK"]: return result self.vo = result["Value"] self.opHelper = Operations(vo=self.vo) useProxy = gConfig.getValue("/LocalSite/StorageElements/%s/UseProxy" % name, False) if not useProxy: useProxy = self.opHelper.getValue("/Services/StorageElements/%s/UseProxy" % name, False) self.valid = True if protocols == None: res = StorageFactory(useProxy).getStorages(name, protocolList=[]) else: res = StorageFactory(useProxy).getStorages(name, protocolList=protocols) if not res["OK"]: self.valid = False self.name = name self.errorReason = res["Message"] else: factoryDict = res["Value"] self.name = factoryDict["StorageName"] self.options = factoryDict["StorageOptions"] self.localProtocols = factoryDict["LocalProtocols"] self.remoteProtocols = factoryDict["RemoteProtocols"] self.storages = factoryDict["StorageObjects"] self.protocolOptions = factoryDict["ProtocolOptions"] self.turlProtocols = factoryDict["TurlProtocols"] self.log = gLogger.getSubLogger("SE[%s]" % self.name) self.readMethods = [ "getFile", "getAccessUrl", "getTransportURL", "prestageFile", "prestageFileStatus", "getDirectory", ] self.writeMethods = [ "retransferOnlineFile", "putFile", "replicateFile", "pinFile", "releaseFile", "createDirectory", "putDirectory", ] self.removeMethods = ["removeFile", "removeDirectory"] self.checkMethods = [ "exists", "getDirectoryMetadata", "getDirectorySize", "getFileSize", "getFileMetadata", "listDirectory", "isDirectory", "isFile", ] self.okMethods = [ "getLocalProtocols", "getPfnForProtocol", "getPfnForLfn", "getPfnPath", "getProtocols", "getRemoteProtocols", "getStorageElementName", "getStorageElementOption", "getStorageParameters", "isLocalSE", ]
def __init__(self, name, plugins=None, vo=None, hideExceptions=False): """ c'tor :param str name: SE name :param list plugins: requested storage plugins :param: vo """ self.methodName = None if vo: self.vo = vo else: result = getVOfromProxyGroup() if not result['OK']: return self.vo = result['Value'] self.opHelper = Operations(vo=self.vo) # These things will soon have to go as well. 'AccessProtocol.1' is all but flexible. proxiedProtocols = gConfig.getValue( '/LocalSite/StorageElements/ProxyProtocols', "").split(',') self.useProxy = (gConfig.getValue( "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name, "UnknownProtocol") in proxiedProtocols) if not self.useProxy: self.useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False) if not self.useProxy: self.useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False) self.valid = True if plugins == None: res = StorageFactory(useProxy=self.useProxy, vo=self.vo).getStorages( name, pluginList=[], hideExceptions=hideExceptions) else: res = StorageFactory(useProxy=self.useProxy, vo=self.vo).getStorages( name, pluginList=plugins, hideExceptions=hideExceptions) if not res['OK']: self.valid = False self.name = name self.errorReason = res['Message'] else: factoryDict = res['Value'] self.name = factoryDict['StorageName'] self.options = factoryDict['StorageOptions'] self.localPlugins = factoryDict['LocalPlugins'] self.remotePlugins = factoryDict['RemotePlugins'] self.storages = factoryDict['StorageObjects'] self.protocolOptions = factoryDict['ProtocolOptions'] self.turlProtocols = factoryDict['TurlProtocols'] for storage in self.storages: storage.setStorageElement(self) self.log = gLogger.getSubLogger("SE[%s]" % self.name) if self.valid: self.useCatalogURL = gConfig.getValue( '/Resources/StorageElements/%s/UseCatalogURL' % self.name, False) self.log.debug("useCatalogURL: %s" % self.useCatalogURL) self.__dmsHelper = DMSHelpers(vo=vo) # Allow SE to overwrite general operation config accessProto = self.options.get('AccessProtocols') self.localAccessProtocolList = accessProto if accessProto else self.__dmsHelper.getAccessProtocols( ) self.log.debug("localAccessProtocolList %s" % self.localAccessProtocolList) writeProto = self.options.get('WriteProtocols') self.localWriteProtocolList = writeProto if writeProto else self.__dmsHelper.getWriteProtocols( ) self.log.debug("localWriteProtocolList %s" % self.localWriteProtocolList) # 'getTransportURL', self.readMethods = [ 'getFile', 'prestageFile', 'prestageFileStatus', 'getDirectory' ] self.writeMethods = [ 'retransferOnlineFile', 'putFile', 'replicateFile', 'pinFile', 'releaseFile', 'createDirectory', 'putDirectory' ] self.removeMethods = ['removeFile', 'removeDirectory'] self.checkMethods = [ 'exists', 'getDirectoryMetadata', 'getDirectorySize', 'getFileSize', 'getFileMetadata', 'listDirectory', 'isDirectory', 'isFile', ] self.okMethods = [ 'getLocalProtocols', 'getProtocols', 'getRemoteProtocols', 'getStorageElementName', 'getStorageParameters', 'getTransportURL', 'isLocalSE' ] self.__fileCatalog = None
fromSE = list(seForSeBases) elif not targetSE: targetSE = list(seForSeBases) fromSE = sorted(fromSE) targetSE = sorted(targetSE) gLogger.notice("Using sources: %s" % ','.join(fromSE)) gLogger.notice("Using target: %s" % ','.join(targetSE)) # Now we construct the SE object for each SE that we want to appear ses = {} for se in set(fromSE + targetSE): ses[se] = StorageElement(seForSeBases[se]) ret = getVOfromProxyGroup() if not ret['OK'] or not ret.get('Value', ''): gLogger.error('Aborting, Bad Proxy:', ret.get('Message', 'Proxy does not belong to a VO!')) exit(1) vo = ret['Value'] gLogger.notice('Using the Virtual Organization:', vo) # dummy LFN, still has to follow lfn convention lfn = '/%s/toto.xml' % vo # Create a matrix of protocol src/dest tpMatrix = defaultdict(dict) # For each source and destination, generate the url pair, and the compatible third party protocols for src, dst in ((x, y) for x in fromSE for y in targetSE): res = ses[dst].generateTransferURLsBetweenSEs(lfn, ses[src], thirdPartyProtocols)
def execute(self): """ Main execution function. """ #Have to work out if the module is part of the last step i.e. #user jobs can have any number of steps and we only want #to run the finalization once. currentStep = int(self.step_commons['STEP_NUMBER']) totalSteps = int(self.workflow_commons['TotalSteps']) if currentStep == totalSteps: self.lastStep = True else: self.log.verbose('Current step = %s, total steps of workflow = %s, UserJobFinalization will enable itself only \ at the last workflow step.' % (currentStep, totalSteps)) if not self.lastStep: #Not last step, do nothing, proceed happily. return S_OK() result = self.resolveInputVariables() if not result['OK']: self.log.error("Failed to resolve input parameters:", result['Message']) return result self.log.info('Initializing %s' % self.version) if not self.workflowStatus['OK'] or not self.stepStatus['OK']: ##Something went wrong in the step or the workflow, do nothing. self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No output data upload attempted') self.request.RequestName = 'job_%d_request.xml' % int(self.jobID) self.request.JobID = self.jobID self.request.SourceComponent = "Job_%d" % int(self.jobID) if not self.userOutputData: self.log.info('No user output data is specified for this job, nothing to do') return S_OK('No output data to upload') #Determine the final list of possible output files for the #workflow and all the parameters needed to upload them. outputList = [] possible_files= [] for i in self.userOutputData: files = getGlobbedFiles(i) for possible_file in files: if possible_file in possible_files: #Don't have twice the same file continue outputList.append({'outputDataType' : i.split('.')[-1].upper(),#this would be used to sort the files in different dirs 'outputDataSE' : self.userOutputSE, 'outputFile' : os.path.basename(possible_file)}) possible_files.append(os.path.basename(possible_file)) self.log.info('Constructing user output LFN(s) for %s' % (', '.join(self.userOutputData))) if not self.jobID: self.jobID = 12345 owner = '' if 'Owner' in self.workflow_commons: owner = self.workflow_commons['Owner'] else: res = getCurrentOwner() if not res['OK']: self.log.error('Could not find proxy') return S_ERROR('Could not obtain owner from proxy') owner = res['Value'] vo = '' if self.workflow_commons.has_key('VO'): vo = self.workflow_commons['VO'] else: res = getVOfromProxyGroup() if not res['OK']: self.log.error('Failed finding the VO') return S_ERROR('Could not obtain VO from proxy') vo = res['Value'] result = constructUserLFNs(int(self.jobID), vo, owner, possible_files, self.userOutputPath) if not result['OK']: self.log.error('Could not create user LFNs', result['Message']) return result userOutputLFNs = result['Value'] self.log.verbose('Calling getCandidateFiles( %s, %s)' % (outputList, userOutputLFNs)) result = self.getCandidateFiles(outputList, userOutputLFNs) if not result['OK']: if not self.ignoreapperrors: self.log.error(result['Message']) self.setApplicationStatus(result['Message']) return S_OK() fileDict = result['Value'] result = self.getFileMetadata(fileDict) if not result['OK']: if not self.ignoreapperrors: self.log.error(result['Message']) self.setApplicationStatus(result['Message']) return S_OK() if not result['Value']: if not self.ignoreapperrors: self.log.info('No output data files were determined to be uploaded for this workflow') self.setApplicationStatus('No Output Data Files To Upload') return S_OK() fileMetadata = result['Value'] orderedSEs = self.userOutputSE self.log.info('Ordered list of output SEs is: %s' % (', '.join(orderedSEs))) final = {} for fileName, metadata in fileMetadata.items(): final[fileName] = metadata final[fileName]['resolvedSE'] = orderedSEs #At this point can exit and see exactly what the module will upload if not self.enable: self.log.info('Module is disabled by control flag, would have attempted \ to upload the following files %s' % ', '.join(final.keys())) for fileName, metadata in final.items(): self.log.info('--------%s--------' % fileName) for n, v in metadata.items(): self.log.info('%s = %s' %(n, v)) return S_OK('Module is disabled by control flag') #Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self.request) #One by one upload the files with failover if necessary replication = {} failover = {} uploaded = [] if not self.failoverTest: for fileName, metadata in final.items(): self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName, ', '.join(metadata['resolvedSE']))) replicateSE = '' result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'], metadata['lfn'], metadata['resolvedSE'], fileMetaDict = metadata, fileCatalog = self.userFileCatalog) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) failover[fileName] = metadata else: #Only attempt replication after successful upload lfn = metadata['lfn'] uploaded.append(lfn) seList = metadata['resolvedSE'] if result['Value'].has_key('uploadedSE'): uploadedSE = result['Value']['uploadedSE'] for se in seList: if not se == uploadedSE: replicateSE = se break if replicateSE and lfn: self.log.info('Will attempt to replicate %s to %s' % (lfn, replicateSE)) replication[lfn] = replicateSE else: failover = final cleanUp = False for fileName, metadata in failover.items(): random.shuffle(self.failoverSEs) targetSE = metadata['resolvedSE'][0] metadata['resolvedSE'] = self.failoverSEs result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'], metadata['lfn'], targetSE, self.failoverSEs, fileMetaDict = metadata, fileCatalog = self.userFileCatalog) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) cleanUp = True continue #for users can continue even if one completely fails else: lfn = metadata['lfn'] uploaded.append(lfn) #For files correctly uploaded must report LFNs to job parameters if uploaded: report = ', '.join( uploaded ) self.jobReport.setJobParameter( 'UploadedOutputData', report ) self.request = failoverTransfer.request #If some or all of the files failed to be saved to failover if cleanUp: self.workflow_commons['Request'] = self.request #Leave any uploaded files just in case it is useful for the user #do not try to replicate any files. return S_ERROR('Failed To Upload Output Data') #If there is now at least one replica for uploaded files can trigger replication rm = ReplicaManager() self.log.info('Sleeping for 10 seconds before attempting replication of recently uploaded files') time.sleep(10) for lfn, repSE in replication.items(): result = rm.replicateAndRegister(lfn, repSE, catalog = self.userFileCatalog) if not result['OK']: self.log.info('Replication failed with below error but file already exists in Grid storage with \ at least one replica:\n%s' % (result)) self.workflow_commons['Request'] = self.request self.generateFailoverFile() self.setApplicationStatus('Job Finished Successfully') return S_OK('Output data uploaded')
def main(): global fullMatch global sites Script.registerSwitch("F", "full-match", "Check all the matching criteria", setFullMatch) Script.registerSwitch( "S:", "site=", "Check matching for these sites (comma separated list)", setSites) Script.registerArgument("job_JDL: file with job JDL description") _, args = Script.parseCommandLine(ignoreErrors=True) from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.ConfigurationSystem.Client.Helpers import Resources from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue with open(args[0]) as f: jdl = f.read() # Get the current VO result = getVOfromProxyGroup() if not result["OK"]: gLogger.error("No proxy found, please login") DIRACExit(-1) voName = result["Value"] resultQueues = Resources.getQueues(siteList=sites, community=voName) if not resultQueues["OK"]: gLogger.error("Failed to get CE information") DIRACExit(-1) siteDict = resultQueues["Value"] result = getQueuesResolved(siteDict, {}, checkPlatform=True) if not resultQueues["OK"]: gLogger.error("Failed to get CE information") DIRACExit(-1) queueDict = result["Value"] # get list of usable sites within this cycle resultMask = SiteStatus().getUsableSites() if not resultMask["OK"]: gLogger.error("Failed to get Site mask information") DIRACExit(-1) siteMaskList = resultMask.get("Value", []) rssClient = ResourceStatus() fields = ("Site", "CE", "Queue", "Status", "Match", "Reason") records = [] for queue, queueInfo in queueDict.items(): site = queueInfo["Site"] ce = queueInfo["CEName"] siteStatus = "Active" if site in siteMaskList else "InActive" ceStatus = siteStatus if rssClient.rssFlag: result = rssClient.getElementStatus(ce, "ComputingElement") if result["OK"]: ceStatus = result["Value"][ce]["all"] result = matchQueue(jdl, queueInfo["ParametersDict"], fullMatch=fullMatch) if not result["OK"]: gLogger.error("Failed in getting match data", result["Message"]) DIRACExit(-1) status = "Active" if siteStatus == "Active" and ceStatus == "Active" else "Inactive" if result["Value"]["Match"]: records.append( (site, ce, queueInfo["QueueName"], status, "Yes", "")) else: records.append((site, ce, queueInfo["QueueName"], status, "No", result["Value"]["Reason"])) gLogger.notice( printTable(fields, records, sortField="Site", columnSeparator=" ", printOut=False))
Usage: %s [<options>] """ % Script.scriptName) Script.parseCommandLine() import DIRAC from DIRAC import gConfig, gLogger from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.Core.Utilities.List import sortList from DIRAC.ConfigurationSystem.Client.Helpers.Resources import Resources from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup if __name__ == "__main__": result = getVOfromProxyGroup() if not result['OK']: gLogger.notice('Error:', result['Message']) DIRAC.exit(1) vo = result['Value'] resources = Resources(vo=vo) result = resources.getEligibleStorageElements() if not result['OK']: gLogger.notice('Error:', result['Message']) DIRAC.exit(2) seList = sortList(result['Value']) resourceStatus = ResourceStatus() result = resourceStatus.getStorageStatus(seList) if not result['OK']:
def __init__(self, catalogs=None, vo=None): """Default constructor""" self.valid = True self.timeout = 180 self.ro_methods = set() self.write_methods = set() self.no_lfn_methods = set() self.readCatalogs = [] self.writeCatalogs = [] self.rootConfigPath = "/Resources/FileCatalogs" self.vo = vo if vo else getVOfromProxyGroup().get("Value", None) self.log = gLogger.getSubLogger(self.__class__.__name__) self.opHelper = Operations(vo=self.vo) catalogList = [] if isinstance(catalogs, six.string_types): catalogList = [catalogs] elif isinstance(catalogs, (list, tuple)): catalogList = list(catalogs) if catalogList: result = self._getEligibleCatalogs() if not result["OK"]: self.log.error("Failed to get eligible catalog") return eligibleFileCatalogs = result["Value"] catalogCheck = True for catalog in catalogList: if catalog not in eligibleFileCatalogs: self.log.error("Specified catalog is not eligible", catalog) catalogCheck = False if catalogCheck: result = self._getSelectedCatalogs(catalogList) else: result = S_ERROR("Specified catalog is not eligible") else: result = self._getCatalogs() if not result["OK"]: self.log.error("Failed to create catalog objects") self.valid = False elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0): self.log.error("No catalog object created") self.valid = False result = self.getMasterCatalogNames() masterCatalogs = result["Value"] # There can not be more than one master catalog haveMaster = False if len(masterCatalogs) > 1: self.log.error("More than one master catalog created") self.valid = False elif len(masterCatalogs) == 1: haveMaster = True # Get the list of write methods if haveMaster: # All the write methods must be present in the master _catalogName, oCatalog, _master = self.writeCatalogs[0] _roList, writeList, nolfnList = oCatalog.getInterfaceMethods() self.write_methods.update(writeList) self.no_lfn_methods.update(nolfnList) else: for _catalogName, oCatalog, _master in self.writeCatalogs: _roList, writeList, nolfnList = oCatalog.getInterfaceMethods() self.write_methods.update(writeList) self.no_lfn_methods.update(nolfnList) # Get the list of read methods for _catalogName, oCatalog, _master in self.readCatalogs: roList, _writeList, nolfnList = oCatalog.getInterfaceMethods() self.ro_methods.update(roList) self.no_lfn_methods.update(nolfnList) self.condParser = FCConditionParser(vo=self.vo, ro_methods=self.ro_methods)
def main(): global fullMatch global sites Script.registerSwitch("F", "full-match", "Check all the matching criteria", setFullMatch) Script.registerSwitch( "S:", "site=", "Check matching for these sites (comma separated list)", setSites) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) == 0: gLogger.error("Error: No job description provided") Script.showHelp(exitCode=1) from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.ConfigurationSystem.Client.Helpers import Resources from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue with open(args[0]) as f: jdl = f.read() # Get the current VO result = getVOfromProxyGroup() if not result['OK']: gLogger.error('No proxy found, please login') DIRACExit(-1) voName = result['Value'] resultQueues = Resources.getQueues(siteList=sites, community=voName) if not resultQueues['OK']: gLogger.error('Failed to get CE information') DIRACExit(-1) siteDict = resultQueues['Value'] result = getQueuesResolved(siteDict) if not resultQueues['OK']: gLogger.error('Failed to get CE information') DIRACExit(-1) queueDict = result['Value'] # get list of usable sites within this cycle resultMask = SiteStatus().getUsableSites() if not resultMask['OK']: gLogger.error('Failed to get Site mask information') DIRACExit(-1) siteMaskList = resultMask.get('Value', []) rssClient = ResourceStatus() fields = ('Site', 'CE', 'Queue', 'Status', 'Match', 'Reason') records = [] for queue, queueInfo in queueDict.items(): site = queueInfo['Site'] ce = queueInfo['CEName'] siteStatus = "Active" if site in siteMaskList else "InActive" ceStatus = siteStatus if rssClient.rssFlag: result = rssClient.getElementStatus(ce, "ComputingElement") if result['OK']: ceStatus = result['Value'][ce]['all'] result = matchQueue(jdl, queueInfo, fullMatch=fullMatch) if not result['OK']: gLogger.error('Failed in getting match data', result['Message']) DIRACExit(-1) status = "Active" if siteStatus == "Active" and ceStatus == "Active" else "Inactive" if result['Value']['Match']: records.append((site, ce, queueInfo['Queue'], status, 'Yes', '')) else: records.append((site, ce, queueInfo['Queue'], status, 'No', result['Value']['Reason'])) gLogger.notice( printTable(fields, records, sortField='Site', columnSeparator=' ', printOut=False))
def __init__(self, **options): """ Constructor. Takes options defined in Resources and Operations for this client. :param options: options dict """ self.diracScopeAlg = options.get("DiracScopeAlg", "dirac") self.useDiracCS = False # use a Rucio config file self.convertUnicode = True proxyInfo = {"OK": False} if os.getenv("RUCIO_AUTH_TYPE" ) == "x509_proxy" and not os.getenv("X509_USER_PROXY"): proxyInfo = getProxyInfo(disableVOMS=True) if proxyInfo["OK"]: os.environ["X509_USER_PROXY"] = proxyInfo["Value"]["path"] sLog.debug("X509_USER_PROXY not defined. Using %s" % proxyInfo["Value"]["path"]) try: try: self._client = Client() self.account = self._client.account except (CannotAuthenticate, MissingClientParameter): if os.getenv("RUCIO_AUTH_TYPE") == "x509_proxy": if not proxyInfo["OK"]: proxyInfo = getProxyInfo(disableVOMS=True) if proxyInfo["OK"]: dn = proxyInfo["Value"]["identity"] username = proxyInfo["Value"]["username"] self.account = username sLog.debug( "Switching to account %s mapped to proxy %s" % (username, dn)) try: self._client = Client(account=self.account) self.scopes = self._client.list_scopes() except Exception as err: sLog.error( "Cannot instantiate RucioFileCatalog interface using a config file", "error : %s" % repr(err)) sLog.info(("will try using Dirac CS")) except Exception as err: # instantiate the client w/o a config file sLog.debug( "instantiate the client w/o a config file - take config params from the CS" ) self.useDiracCS = True proxyInfo = getProxyInfo(disableVOMS=True) if proxyInfo["OK"]: proxyDict = proxyInfo["Value"] self.proxyPath = proxyDict.get("path", None) self.username = proxyDict.get("username", None) else: sLog.error("Cannot instantiate RucioFileCatalog interface", proxyInfo["Message"]) return self.VO = getVOfromProxyGroup()["Value"] self.rucioHost = options.get("RucioHost", None) self.authHost = options.get("AuthHost", None) self.caCertPath = Locations.getCAsLocation() try: sLog.info("Logging in with a proxy located at: %s" % self.proxyPath) sLog.debug("account: ", self.username) sLog.debug("rucio host: ", self.rucioHost) sLog.debug("auth host: ", self.authHost) sLog.debug("CA cert path: ", self.caCertPath) sLog.debug("VO: ", self.VO) self._client = Client( account=self.username, rucio_host=self.rucioHost, auth_host=self.authHost, ca_cert=self.caCertPath, auth_type="x509_proxy", creds={"client_proxy": self.proxyPath}, timeout=600, user_agent="rucio-clients", vo=self.VO, ) sLog.debug( "Rucio client instantiated successfully for VO %s and account %s " % (self.VO, self.username)) except Exception as err: sLog.error("Cannot instantiate RucioFileCatalog interface", "error : %s" % repr(err))
def __init__( self, catalogs = None, vo = None ): """ Default constructor """ self.valid = True self.timeout = 180 self.ro_methods = set() self.write_methods = set() self.no_lfn_methods = set() self.readCatalogs = [] self.writeCatalogs = [] self.rootConfigPath = '/Resources/FileCatalogs' self.vo = vo if vo else getVOfromProxyGroup().get( 'Value', None ) self.log = gLogger.getSubLogger( "FileCatalog" ) self.opHelper = Operations( vo = self.vo ) catalogList = [] if isinstance( catalogs, basestring ): catalogList = [catalogs] elif isinstance( catalogs, ( list, tuple ) ): catalogList = list( catalogs ) if catalogList: result = self._getEligibleCatalogs() if not result['OK']: self.log.error( "Failed to get eligible catalog" ) return eligibleFileCatalogs = result['Value'] catalogCheck = True for catalog in catalogList: if catalog not in eligibleFileCatalogs: self.log.error( "Specified catalog is not eligible", catalog ) catalogCheck = False if catalogCheck: result = self._getSelectedCatalogs( catalogList ) else: result = S_ERROR( "Specified catalog is not eligible" ) else: result = self._getCatalogs() if not result['OK']: self.log.error( "Failed to create catalog objects" ) self.valid = False elif ( len( self.readCatalogs ) == 0 ) and ( len( self.writeCatalogs ) == 0 ): self.log.error( "No catalog object created" ) self.valid = False result = self.getMasterCatalogNames() masterCatalogs = result['Value'] # There can not be more than one master catalog haveMaster = False if len( masterCatalogs ) > 1: self.log.error( "More than one master catalog created" ) self.valid = False elif len( masterCatalogs ) == 1: haveMaster = True # Get the list of write methods if haveMaster: # All the write methods must be present in the master _catalogName, oCatalog, _master = self.writeCatalogs[0] _roList, writeList, nolfnList = oCatalog.getInterfaceMethods() self.write_methods.update( writeList ) self.no_lfn_methods.update( nolfnList ) else: for _catalogName, oCatalog, _master in self.writeCatalogs: _roList, writeList, nolfnList = oCatalog.getInterfaceMethods() self.write_methods.update( writeList ) self.no_lfn_methods.update( nolfnList ) # Get the list of read methods for _catalogName, oCatalog, _master in self.readCatalogs: roList, _writeList, nolfnList = oCatalog.getInterfaceMethods() self.ro_methods.update( roList ) self.no_lfn_methods.update( nolfnList ) self.condParser = FCConditionParser( vo = self.vo, ro_methods = self.ro_methods )
def __init__( self, name, protocols = None, vo = None ): """ c'tor :param str name: SE name :param list protocols: requested protocols """ self.vo = vo if not vo: result = getVOfromProxyGroup() if not result['OK']: return result self.vo = result['Value'] self.opHelper = Operations( vo = self.vo ) useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False ) if not useProxy: useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False ) self.valid = True if protocols == None: res = StorageFactory( useProxy ).getStorages( name, protocolList = [] ) else: res = StorageFactory( useProxy ).getStorages( name, protocolList = protocols ) if not res['OK']: self.valid = False self.name = name self.errorReason = res['Message'] else: factoryDict = res['Value'] self.name = factoryDict['StorageName'] self.options = factoryDict['StorageOptions'] self.localProtocols = factoryDict['LocalProtocols'] self.remoteProtocols = factoryDict['RemoteProtocols'] self.storages = factoryDict['StorageObjects'] self.protocolOptions = factoryDict['ProtocolOptions'] self.turlProtocols = factoryDict['TurlProtocols'] self.log = gLogger.getSubLogger( "SE[%s]" % self.name ) self.readMethods = [ 'getFile', 'getAccessUrl', 'getTransportURL', 'prestageFile', 'prestageFileStatus', 'getDirectory'] self.writeMethods = [ 'retransferOnlineFile', 'putFile', 'replicateFile', 'pinFile', 'releaseFile', 'createDirectory', 'putDirectory' ] self.removeMethods = [ 'removeFile', 'removeDirectory' ] self.checkMethods = [ 'exists', 'getDirectoryMetadata', 'getDirectorySize', 'getFileSize', 'getFileMetadata', 'getLocalProtocols', 'getPfnForProtocol', 'getPfnForLfn', 'getPfnPath', 'getProtocols', 'getRemoteProtocols', 'getStorageElementName', 'getStorageElementOption', 'getStorageParameters', 'listDirectory', 'isDirectory', 'isFile', 'isLocalSE' ]
final_dict[site].update({ "Tags": ldapTag(ce, vo), "CE": curr_ces }) #final_dict[site]={"Tags":ldapTag(ce,vo),"CE":[ce]} ret_dict = {} for key in final_dict: if len(final_dict[key]['CE']) != 0: ret_dict[key] = final_dict[key] return ret_dict if __name__ == "__main__": Script.parseCommandLine() vo = "glast.org" res = getVOfromProxyGroup() if not res['OK']: gLogger.error(res['Message']) gLogger.error('Could not get VO from CS, assuming glast.org') dexit(1) else: gLogger.info(res) vo = res['Value'] d = main(vo) for key in d: print('Name of DIRAC site %s\nName of CEs: %s' % (key, str(d[key]["CE"]))) for tag in d[key]["Tags"]: print('\t%s' % tag) print('\n')
def getCEStatus(self): """ Method to return information on running and pending jobs. We hope to satisfy both instances that use robot proxies and those which use proper configurations. """ result = self._prepareProxy() if not result['OK']: self.log.error('ARCComputingElement: failed to set up proxy', result['Message']) return result self.usercfg.ProxyPath(os.environ['X509_USER_PROXY']) # Try to find out which VO we are running for. vo = '' res = getVOfromProxyGroup() if res['OK']: vo = res['Value'] result = S_OK() result['SubmittedJobs'] = 0 if not vo: # Presumably the really proper way forward once the infosys-discuss WG comes up with a solution # and it is implemented. Needed for DIRAC instances which use robot certificates for pilots. endpoints = [arc.Endpoint(str("ldap://" + self.ceHost + "/MDS-Vo-name=local,o=grid"), arc.Endpoint.COMPUTINGINFO, 'org.nordugrid.ldapng')] retriever = arc.ComputingServiceRetriever(self.usercfg, endpoints) retriever.wait() # Takes a bit of time to get and parse the ldap information targets = retriever.GetExecutionTargets() ceStats = targets[0].ComputingShare self.log.debug("Running jobs for CE %s : %s" % (self.ceHost, ceStats.RunningJobs)) self.log.debug("Waiting jobs for CE %s : %s" % (self.ceHost, ceStats.WaitingJobs)) result['RunningJobs'] = ceStats.RunningJobs result['WaitingJobs'] = ceStats.WaitingJobs else: # The system which works properly at present for ARC CEs that are configured correctly. # But for this we need the VO to be known - ask me (Raja) for the whole story if interested. # cmd = 'ldapsearch -x -LLL -H ldap://%s:2135 -b mds-vo-name=resource,o=grid "(GlueVOViewLocalID=%s)"' % ( # self.ceHost, vo.lower()) if not self.queue: self.log.error('ARCComputingElement: No queue ...') res = S_ERROR('Unknown queue (%s) failure for site %s' % (self.queue, self.ceHost)) return res cmd1 = "ldapsearch -x -o ldif-wrap=no -LLL -h %s:2135 -b \'o=glue\' " % self.ceHost cmd2 = '"(&(objectClass=GLUE2MappingPolicy)(GLUE2PolicyRule=vo:%s))"' % vo.lower() cmd3 = ' | grep GLUE2MappingPolicyShareForeignKey | grep %s' % (self.queue.split("-")[-1]) cmd4 = ' | sed \'s/GLUE2MappingPolicyShareForeignKey: /GLUE2ShareID=/\' ' cmd5 = ' | xargs -L1 ldapsearch -x -o ldif-wrap=no -LLL -h %s:2135 -b \'o=glue\' ' % self.ceHost cmd6 = ' | egrep \'(ShareWaiting|ShareRunning)\'' res = shellCall(0, cmd1 + cmd2 + cmd3 + cmd4 + cmd5 + cmd6) if not res['OK']: self.log.debug("Could not query CE %s - is it down?" % self.ceHost) return res try: ldapValues = res['Value'][1].split("\n") running = [lValue for lValue in ldapValues if 'GLUE2ComputingShareRunningJobs' in lValue] waiting = [lValue for lValue in ldapValues if 'GLUE2ComputingShareWaitingJobs' in lValue] result['RunningJobs'] = int(running[0].split(":")[1]) result['WaitingJobs'] = int(waiting[0].split(":")[1]) except IndexError: res = S_ERROR('Unknown ldap failure for site %s' % self.ceHost) return res return result
def __init__(self, name, plugins=None, vo=None, hideExceptions=False): """ c'tor :param str name: SE name :param list plugins: requested storage plugins :param vo: vo """ self.methodName = None if vo: self.vo = vo else: result = getVOfromProxyGroup() if not result['OK']: return self.vo = result['Value'] self.opHelper = Operations(vo=self.vo) # These things will soon have to go as well. 'AccessProtocol.1' is all but flexible. proxiedProtocols = gConfig.getValue('/LocalSite/StorageElements/ProxyProtocols', "").split(',') self.useProxy = ( gConfig.getValue( "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name, "UnknownProtocol") in proxiedProtocols) if not self.useProxy: self.useProxy = gConfig.getValue('/LocalSite/StorageElements/%s/UseProxy' % name, False) if not self.useProxy: self.useProxy = self.opHelper.getValue('/Services/StorageElements/%s/UseProxy' % name, False) self.valid = True if plugins is None: res = StorageFactory( useProxy=self.useProxy, vo=self.vo).getStorages( name, pluginList=[], hideExceptions=hideExceptions) else: res = StorageFactory( useProxy=self.useProxy, vo=self.vo).getStorages( name, pluginList=plugins, hideExceptions=hideExceptions) if not res['OK']: self.valid = False self.name = name self.errorReason = res['Message'] else: factoryDict = res['Value'] self.name = factoryDict['StorageName'] self.options = factoryDict['StorageOptions'] self.localPlugins = factoryDict['LocalPlugins'] self.remotePlugins = factoryDict['RemotePlugins'] self.storages = factoryDict['StorageObjects'] self.protocolOptions = factoryDict['ProtocolOptions'] self.turlProtocols = factoryDict['TurlProtocols'] for storage in self.storages: storage.setStorageElement(self) self.log = gLogger.getSubLogger("SE[%s]" % self.name) if self.valid: self.useCatalogURL = gConfig.getValue( '/Resources/StorageElements/%s/UseCatalogURL' % self.name, False) self.log.debug("useCatalogURL: %s" % self.useCatalogURL) self.__dmsHelper = DMSHelpers(vo=vo) # Allow SE to overwrite general operation config accessProto = self.options.get('AccessProtocols') self.localAccessProtocolList = accessProto if accessProto else self.__dmsHelper.getAccessProtocols() self.log.debug("localAccessProtocolList %s" % self.localAccessProtocolList) writeProto = self.options.get('WriteProtocols') self.localWriteProtocolList = writeProto if writeProto else self.__dmsHelper.getWriteProtocols() self.log.debug("localWriteProtocolList %s" % self.localWriteProtocolList) # 'getTransportURL', self.readMethods = ['getFile', 'prestageFile', 'prestageFileStatus', 'getDirectory'] self.writeMethods = ['retransferOnlineFile', 'putFile', 'replicateFile', 'pinFile', 'releaseFile', 'createDirectory', 'putDirectory'] self.removeMethods = ['removeFile', 'removeDirectory'] self.checkMethods = ['exists', 'getDirectoryMetadata', 'getDirectorySize', 'getFileSize', 'getFileMetadata', 'listDirectory', 'isDirectory', 'isFile', 'getOccupancy' ] self.okMethods = ['getLocalProtocols', 'getProtocols', 'getRemoteProtocols', 'storageElementName', 'getStorageParameters', 'getTransportURL', 'isLocalSE'] self.__fileCatalog = None
def __init__(self, name, protocols=None, vo=None): """ c'tor :param str name: SE name :param list protocols: requested protocols :param vo """ self.methodName = None if vo: self.vo = vo else: result = getVOfromProxyGroup() if not result['OK']: return self.vo = result['Value'] self.opHelper = Operations(vo=self.vo) proxiedProtocols = gConfig.getValue( '/LocalSite/StorageElements/ProxyProtocols', "").split(',') useProxy = (gConfig.getValue( "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name, "UnknownProtocol") in proxiedProtocols) if not useProxy: useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False) if not useProxy: useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False) self.valid = True if protocols == None: res = StorageFactory(useProxy=useProxy, vo=self.vo).getStorages(name, protocolList=[]) else: res = StorageFactory(useProxy=useProxy, vo=self.vo).getStorages( name, protocolList=protocols) if not res['OK']: self.valid = False self.name = name self.errorReason = res['Message'] else: factoryDict = res['Value'] self.name = factoryDict['StorageName'] self.options = factoryDict['StorageOptions'] self.localProtocols = factoryDict['LocalProtocols'] self.remoteProtocols = factoryDict['RemoteProtocols'] self.storages = factoryDict['StorageObjects'] self.protocolOptions = factoryDict['ProtocolOptions'] self.turlProtocols = factoryDict['TurlProtocols'] self.log = gLogger.getSubLogger("SE[%s]" % self.name) self.readMethods = [ 'getFile', 'getAccessUrl', 'getTransportURL', 'prestageFile', 'prestageFileStatus', 'getDirectory' ] self.writeMethods = [ 'retransferOnlineFile', 'putFile', 'replicateFile', 'pinFile', 'releaseFile', 'createDirectory', 'putDirectory' ] self.removeMethods = ['removeFile', 'removeDirectory'] self.checkMethods = [ 'exists', 'getDirectoryMetadata', 'getDirectorySize', 'getFileSize', 'getFileMetadata', 'listDirectory', 'isDirectory', 'isFile', ] self.okMethods = [ 'getLocalProtocols', 'getPfnForProtocol', 'getPfnForLfn', 'getPfnPath', 'getProtocols', 'getRemoteProtocols', 'getStorageElementName', 'getStorageElementOption', 'getStorageParameters', 'isLocalSE' ]
def main(): from DIRAC.Core.Base import Script Script.registerSwitch('', 'FromSE=', 'SE1[,SE2,...]') Script.registerSwitch('', 'TargetSE=', 'SE1[,SE2,...]') Script.registerSwitch( '', 'OutputFile=', 'CSV output file (default /tmp/protocol-matrix.csv)') Script.registerSwitch( '', 'Bidirection', 'If FromSE or TargetSE are specified, make a square matrix ') Script.registerSwitch('', 'FTSOnly', 'Only display the protocols sent to FTS') Script.registerSwitch('', 'ExcludeSE=', 'SEs to not take into account for the matrix') from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from DIRAC import gConfig, gLogger from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup fromSE = [] targetSE = [] excludeSE = [] outputFile = '/tmp/protocol-matrix.csv' bidirection = False ftsOnly = False for switch in Script.getUnprocessedSwitches(): if switch[0] == 'FromSE': fromSE = switch[1].split(',') elif switch[0] == 'TargetSE': targetSE = switch[1].split(',') elif switch[0] == 'ExcludeSE': excludeSE = switch[1].split(',') elif switch[0] == 'OutputFile': outputFile = switch[1] elif switch[0] == 'Bidirection': bidirection = True elif switch[0] == 'FTSOnly': ftsOnly = True thirdPartyProtocols = DMSHelpers().getThirdPartyProtocols() # List all the BaseSE seBases = gConfig.getSections('/Resources/StorageElementBases')['Value'] # construct a dict { baseSE : <inherited storages>} seForSeBases = {} allSEs = gConfig.getSections('/Resources/StorageElements/')['Value'] # Remove the SEs that we want to exclude allSEs = set(allSEs) - set(excludeSE) # We go through all the SEs and fill in the seForSEBases dict. # Basically, at the end of the loop, the dict will contain # for each baseSE an entry corresponding to one real storage (the first one) # and itself for each real non inherited SE for se in allSEs: baseSE = gConfig.getOption('/Resources/StorageElements/%s/BaseSE' % se).get('Value') if baseSE: if baseSE not in seForSeBases: seForSeBases[baseSE] = se else: # If no baseSE, we put self seForSeBases[se] = se # Now let's take into account what source and destination we want. # If the user did not specify source or dest, generate everything if not fromSE and not targetSE: fromSE = list(seForSeBases) targetSE = list(seForSeBases) else: # he specified at least source of dest # if bidirection, source and target should be the same if bidirection: if not fromSE and targetSE: # we gave target, but no source fromSE = targetSE elif fromSE and not targetSE: # we gave source but no target targetSE = fromSE elif fromSE and targetSE: # we gave both fromSE = targetSE = list(set(fromSE + targetSE)) else: # no bidirection # he specified a targetSE if not fromSE: fromSE = list(seForSeBases) elif not targetSE: targetSE = list(seForSeBases) fromSE = sorted(fromSE) targetSE = sorted(targetSE) gLogger.notice("Using sources: %s" % ','.join(fromSE)) gLogger.notice("Using target: %s" % ','.join(targetSE)) # Now we construct the SE object for each SE that we want to appear ses = {} for se in set(fromSE + targetSE): ses[se] = StorageElement(seForSeBases.get(se, se)) ret = getVOfromProxyGroup() if not ret['OK'] or not ret.get('Value', ''): gLogger.error('Aborting, Bad Proxy:', ret.get('Message', 'Proxy does not belong to a VO!')) exit(1) vo = ret['Value'] gLogger.notice('Using the Virtual Organization:', vo) # dummy LFN, still has to follow lfn convention lfn = '/%s/toto.xml' % vo # Create a matrix of protocol src/dest tpMatrix = defaultdict(dict) # For each source and destination, generate the url pair, and the compatible third party protocols for src, dst in ((x, y) for x in fromSE for y in targetSE): res = ses[dst].generateTransferURLsBetweenSEs(lfn, ses[src], thirdPartyProtocols) if not res['OK']: surls = 'Error' gLogger.notice( "Could not generate transfer URLS", "src:%s, dst:%s, error:%s" % (src, dst, res['Message'])) else: # We only keep the protocol part of the url surls = '/'.join(res['Value']['Protocols']) # Add also the third party protocols proto = ','.join(ses[dst].negociateProtocolWithOtherSE( ses[src], thirdPartyProtocols)['Value']) if ftsOnly: tpMatrix[src][dst] = '%s' % surls else: tpMatrix[src][dst] = '%s (%s)' % (surls, proto) gLogger.verbose("%s -> %s: %s" % (src, dst, surls)) gLogger.verbose("%s -> %s: %s" % (src, dst, proto)) # Write the matrix in the file with open(outputFile, 'wb') as csvfile: csvWriter = csv.writer(csvfile, delimiter=';', quoting=csv.QUOTE_MINIMAL) csvWriter.writerow(['src/dst'] + targetSE) for src in fromSE: srcRow = [src] for dst in targetSE: srcRow.append(tpMatrix[src].get(dst, 'NA')) csvWriter.writerow(srcRow) gLogger.notice('Wrote Matrix to', outputFile)
def __init__( self, name, plugins = None, vo = None, hideExceptions = False ): """ c'tor :param str name: SE name :param list plugins: requested storage plugins :param: vo """ self.methodName = None if vo: self.vo = vo else: result = getVOfromProxyGroup() if not result['OK']: return self.vo = result['Value'] self.opHelper = Operations( vo = self.vo ) proxiedProtocols = gConfig.getValue( '/LocalSite/StorageElements/ProxyProtocols', "" ).split( ',' ) useProxy = ( gConfig.getValue( "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name, "UnknownProtocol" ) in proxiedProtocols ) if not useProxy: useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False ) if not useProxy: useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False ) self.valid = True if plugins == None: res = StorageFactory( useProxy = useProxy, vo = self.vo ).getStorages( name, pluginList = [], hideExceptions = hideExceptions ) else: res = StorageFactory( useProxy = useProxy, vo = self.vo ).getStorages( name, pluginList = plugins, hideExceptions = hideExceptions ) if not res['OK']: self.valid = False self.name = name self.errorReason = res['Message'] else: factoryDict = res['Value'] self.name = factoryDict['StorageName'] self.options = factoryDict['StorageOptions'] self.localPlugins = factoryDict['LocalPlugins'] self.remotePlugins = factoryDict['RemotePlugins'] self.storages = factoryDict['StorageObjects'] self.protocolOptions = factoryDict['ProtocolOptions'] self.turlProtocols = factoryDict['TurlProtocols'] for storage in self.storages: storage.setStorageElement( self ) self.log = gLogger.getSubLogger( "SE[%s]" % self.name ) self.useCatalogURL = gConfig.getValue( '/Resources/StorageElements/%s/UseCatalogURL' % self.name, False ) # 'getTransportURL', self.readMethods = [ 'getFile', 'prestageFile', 'prestageFileStatus', 'getDirectory'] self.writeMethods = [ 'retransferOnlineFile', 'putFile', 'replicateFile', 'pinFile', 'releaseFile', 'createDirectory', 'putDirectory' ] self.removeMethods = [ 'removeFile', 'removeDirectory' ] self.checkMethods = [ 'exists', 'getDirectoryMetadata', 'getDirectorySize', 'getFileSize', 'getFileMetadata', 'listDirectory', 'isDirectory', 'isFile', ] self.okMethods = [ 'getLocalProtocols', 'getProtocols', 'getRemoteProtocols', 'getStorageElementName', 'getStorageParameters', 'getTransportURL', 'isLocalSE' ] self.__fileCatalog = None
def execute(self): """ Main execution function. """ #Have to work out if the module is part of the last step i.e. #user jobs can have any number of steps and we only want #to run the finalization once. currentStep = int(self.step_commons['STEP_NUMBER']) totalSteps = int(self.workflow_commons['TotalSteps']) if currentStep == totalSteps: self.lastStep = True else: self.log.verbose( 'Current step = %s, total steps of workflow = %s, UserJobFinalization will enable itself only \ at the last workflow step.' % (currentStep, totalSteps)) if not self.lastStep: #Not last step, do nothing, proceed happily. return S_OK() result = self.resolveInputVariables() if not result['OK']: self.log.error("Failed to resolve input parameters:", result['Message']) return result self.log.info('Initializing %s' % self.version) if not self.workflowStatus['OK'] or not self.stepStatus['OK']: ##Something went wrong in the step or the workflow, do nothing. self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No output data upload attempted') self.request.RequestName = 'job_%d_request.xml' % int(self.jobID) self.request.JobID = self.jobID self.request.SourceComponent = "Job_%d" % int(self.jobID) if not self.userOutputData: self.log.info( 'No user output data is specified for this job, nothing to do') return S_OK('No output data to upload') #Determine the final list of possible output files for the #workflow and all the parameters needed to upload them. outputList = [] possible_files = [] for i in self.userOutputData: files = getGlobbedFiles(i) for possible_file in files: if possible_file in possible_files: #Don't have twice the same file continue outputList.append({ 'outputDataType': i.split('.')[-1].upper( ), #this would be used to sort the files in different dirs 'outputDataSE': self.userOutputSE, 'outputFile': os.path.basename(possible_file) }) possible_files.append(os.path.basename(possible_file)) self.log.info('Constructing user output LFN(s) for %s' % (', '.join(self.userOutputData))) if not self.jobID: self.jobID = 12345 owner = '' if 'Owner' in self.workflow_commons: owner = self.workflow_commons['Owner'] else: res = getCurrentOwner() if not res['OK']: self.log.error('Could not find proxy') return S_ERROR('Could not obtain owner from proxy') owner = res['Value'] vo = '' if self.workflow_commons.has_key('VO'): vo = self.workflow_commons['VO'] else: res = getVOfromProxyGroup() if not res['OK']: self.log.error('Failed finding the VO') return S_ERROR('Could not obtain VO from proxy') vo = res['Value'] result = constructUserLFNs(int(self.jobID), vo, owner, possible_files, self.userOutputPath) if not result['OK']: self.log.error('Could not create user LFNs', result['Message']) return result userOutputLFNs = result['Value'] self.log.verbose('Calling getCandidateFiles( %s, %s)' % (outputList, userOutputLFNs)) result = self.getCandidateFiles(outputList, userOutputLFNs) if not result['OK']: if not self.ignoreapperrors: self.log.error(result['Message']) self.setApplicationStatus(result['Message']) return S_OK() fileDict = result['Value'] result = self.getFileMetadata(fileDict) if not result['OK']: if not self.ignoreapperrors: self.log.error(result['Message']) self.setApplicationStatus(result['Message']) return S_OK() if not result['Value']: if not self.ignoreapperrors: self.log.info( 'No output data files were determined to be uploaded for this workflow' ) self.setApplicationStatus('No Output Data Files To Upload') return S_OK() fileMetadata = result['Value'] orderedSEs = self.userOutputSE self.log.info('Ordered list of output SEs is: %s' % (', '.join(orderedSEs))) final = {} for fileName, metadata in fileMetadata.items(): final[fileName] = metadata final[fileName]['resolvedSE'] = orderedSEs #At this point can exit and see exactly what the module will upload if not self.enable: self.log.info( 'Module is disabled by control flag, would have attempted \ to upload the following files %s' % ', '.join(final.keys())) for fileName, metadata in final.items(): self.log.info('--------%s--------' % fileName) for n, v in metadata.items(): self.log.info('%s = %s' % (n, v)) return S_OK('Module is disabled by control flag') #Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self.request) #One by one upload the files with failover if necessary replication = {} failover = {} uploaded = [] if not self.failoverTest: for fileName, metadata in final.items(): self.log.info( "Attempting to store file %s to the following SE(s):\n%s" % (fileName, ', '.join(metadata['resolvedSE']))) replicateSE = '' result = failoverTransfer.transferAndRegisterFile( fileName, metadata['localpath'], metadata['lfn'], metadata['resolvedSE'], fileMetaDict=metadata, fileCatalog=self.userFileCatalog) if not result['OK']: self.log.error( 'Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) failover[fileName] = metadata else: #Only attempt replication after successful upload lfn = metadata['lfn'] uploaded.append(lfn) seList = metadata['resolvedSE'] if result['Value'].has_key('uploadedSE'): uploadedSE = result['Value']['uploadedSE'] for se in seList: if not se == uploadedSE: replicateSE = se break if replicateSE and lfn: self.log.info('Will attempt to replicate %s to %s' % (lfn, replicateSE)) replication[lfn] = replicateSE else: failover = final cleanUp = False for fileName, metadata in failover.items(): random.shuffle(self.failoverSEs) targetSE = metadata['resolvedSE'][0] metadata['resolvedSE'] = self.failoverSEs result = failoverTransfer.transferAndRegisterFileFailover( fileName, metadata['localpath'], metadata['lfn'], targetSE, self.failoverSEs, fileMetaDict=metadata, fileCatalog=self.userFileCatalog) if not result['OK']: self.log.error( 'Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) cleanUp = True continue #for users can continue even if one completely fails else: lfn = metadata['lfn'] uploaded.append(lfn) #For files correctly uploaded must report LFNs to job parameters if uploaded: report = ', '.join(uploaded) self.jobReport.setJobParameter('UploadedOutputData', report) self.request = failoverTransfer.request #If some or all of the files failed to be saved to failover if cleanUp: self.workflow_commons['Request'] = self.request #Leave any uploaded files just in case it is useful for the user #do not try to replicate any files. return S_ERROR('Failed To Upload Output Data') #If there is now at least one replica for uploaded files can trigger replication rm = ReplicaManager() self.log.info( 'Sleeping for 10 seconds before attempting replication of recently uploaded files' ) time.sleep(10) for lfn, repSE in replication.items(): result = rm.replicateAndRegister(lfn, repSE, catalog=self.userFileCatalog) if not result['OK']: self.log.info( 'Replication failed with below error but file already exists in Grid storage with \ at least one replica:\n%s' % (result)) self.workflow_commons['Request'] = self.request self.generateFailoverFile() self.setApplicationStatus('Job Finished Successfully') return S_OK('Output data uploaded')
def fts3Transfer(self): """replicate and register using FTS3""" self.log.info("scheduling files in FTS3...") # Check first if we do not have ongoing transfers res = self._checkExistingFTS3Operations() if not res["OK"]: return res # if res['Value'] is False # it means that there are ongoing transfers # and we should stop here if res["Value"] is False: # return S_OK such that the request is put back return S_OK() fts3Files = [] toSchedule = {} # Dict which maps the FileID to the object rmsFilesIds = {} if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Attempted", len(self.getWaitingFilesList()))) for opFile in self.getWaitingFilesList(): rmsFilesIds[opFile.FileID] = opFile opFile.Error = "" if not self.rmsMonitoring: gMonitor.addMark("FTSScheduleAtt") # # check replicas replicas = self._filterReplicas(opFile) if not replicas["OK"]: continue replicas = replicas["Value"] validReplicas = replicas["Valid"] noMetaReplicas = replicas["NoMetadata"] noReplicas = replicas["NoReplicas"] badReplicas = replicas["Bad"] noPFN = replicas["NoPFN"] if validReplicas: validTargets = list( set(self.operation.targetSEList) - set(validReplicas)) if not validTargets: self.log.info("file %s is already present at all targets" % opFile.LFN) opFile.Status = "Done" else: toSchedule[opFile.LFN] = [opFile, validTargets] else: if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Failed", 1)) else: gMonitor.addMark("FTSScheduleFail") if noMetaReplicas: self.log.warn( "unable to schedule file", "'%s': couldn't get metadata at %s" % (opFile.LFN, ",".join(noMetaReplicas)), ) opFile.Error = "Couldn't get metadata" elif noReplicas: self.log.error( "Unable to schedule transfer", "File %s doesn't exist at %s" % (opFile.LFN, ",".join(noReplicas)), ) opFile.Error = "No replicas found" opFile.Status = "Failed" elif badReplicas: self.log.error( "Unable to schedule transfer", "File %s, all replicas have a bad checksum at %s" % (opFile.LFN, ",".join(badReplicas)), ) opFile.Error = "All replicas have a bad checksum" opFile.Status = "Failed" elif noPFN: self.log.warn( "unable to schedule %s, could not get a PFN at %s" % (opFile.LFN, ",".join(noPFN))) if self.rmsMonitoring: self.rmsMonitoringReporter.commit() res = self._addMetadataToFiles(toSchedule) if not res["OK"]: return res else: filesToSchedule = res["Value"] for lfn in filesToSchedule: opFile = filesToSchedule[lfn] validTargets = toSchedule[lfn][1] for targetSE in validTargets: ftsFile = FTS3File.fromRMSFile(opFile, targetSE) fts3Files.append(ftsFile) if fts3Files: res = Registry.getUsernameForDN(self.request.OwnerDN) if not res["OK"]: self.log.error( "Cannot get username for DN", "%s %s" % (self.request.OwnerDN, res["Message"])) return res username = res["Value"] fts3Operation = FTS3TransferOperation.fromRMSObjects( self.request, self.operation, username) fts3Operation.ftsFiles = fts3Files try: if not fts3Operation.activity: vo = getVOfromProxyGroup().get("Value") fts3Plugin = getFTS3Plugin(vo=vo) fts3Operation.activity = fts3Plugin.inferFTSActivity( fts3Operation, self.request, self.operation) except Exception: pass ftsSchedule = FTS3Client().persistOperation(fts3Operation) if not ftsSchedule["OK"]: self.log.error("Completely failed to schedule to FTS3:", ftsSchedule["Message"]) return ftsSchedule # might have nothing to schedule ftsSchedule = ftsSchedule["Value"] self.log.info("Scheduled with FTS3Operation id %s" % ftsSchedule) self.log.info("%d files have been scheduled to FTS3" % len(fts3Files)) if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( self.createRMSRecord("Successful", len(fts3Files))) for ftsFile in fts3Files: opFile = rmsFilesIds[ftsFile.rmsFileID] if not self.rmsMonitoring: gMonitor.addMark("FTSScheduleOK", 1) opFile.Status = "Scheduled" self.log.debug("%s has been scheduled for FTS" % opFile.LFN) else: self.log.info("No files to schedule after metadata checks") if self.rmsMonitoring: self.rmsMonitoringReporter.commit() # Just in case some transfers could not be scheduled, try them with RM return self.dmTransfer(fromFTS=True)