def __init__(self): """ Initialization of module base. """ self.log = gLogger.getSubLogger('ModuleBase') # FIXME: do we need to do this for every module? result = getProxyInfoAsString() if not result['OK']: self.log.error('Could not obtain proxy information in module environment with message:\n', result['Message']) else: self.log.info('Payload proxy information:\n', result['Value']) self.systemConfig = '' self.applicationLog = '' self.applicationVersion='' self.applicationName = '' self.InputData = '' self.SteeringFile = '' self.energy = 0 self.NumberOfEvents = 0 self.result = S_ERROR() self.InputFile = '' self.ignoremissingInput = False self.OutputFile = '' self.jobType = '' self.stdError = '' self.debug = False self.jobID = None if os.environ.has_key('JOBID'): self.jobID = os.environ['JOBID'] self.eventstring = '' self.ignoreapperrors = False
def monitorProxy( self, glexecLocation, pilotProxy, payloadProxy ): """ Monitor the payload proxy and renew as necessary. """ if not os.path.exists( pilotProxy ): return S_ERROR( 'Pilot proxy not found at %s' % pilotProxy ) if not os.path.exists( payloadProxy ): return S_ERROR( 'Payload proxy not found at %s' % payloadProxy ) result = getProxyInfoAsString( payloadProxy ) if not result['OK']: self.log.error( 'Could not get payload proxy info', result ) return result self.log.info( 'Payload proxy information seen from pilot:\n%s' % result['Value'] ) gProxyManager.renewProxy( minLifeTime = self.minProxyTime, newProxyLifeTime = self.defaultProxyTime, proxyToConnect = pilotProxy ) if glexecLocation: self.log.info( 'Rerunning glexec without arguments to renew payload proxy' ) result = self.glexecExecute( None, glexecLocation ) if not result['OK']: self.log.error( result ) else: self.log.info( 'Running without glexec, checking local proxy' ) return S_OK( 'Proxy checked' )
def __sendErrorMail(self, rb, name, command, result, proxy): """ In case or error with RB/WM: - check if RB/WMS still in use - remove RB/WMS from current list - check if RB/WMS not in cache - add RB/WMS to cache - send Error mail """ if rb in self.resourceBrokers: try: self.resourceBrokers.remove(rb) self.log.info("Removed RB from list", rb) except: pass if not self.__failingWMSCache.exists(rb): self.__failingWMSCache.add(rb, self.errorClearTime) # disable for 30 minutes mailAddress = self.errorMailAddress msg = "" if not result["OK"]: subject = "%s: timeout executing %s" % (rb, name) msg += "\n%s" % result["Message"] elif result["Value"][0] != 0: if re.search("the server is temporarily drained", " ".join(result["Value"][1:3])): return if re.search("System load is too high:", " ".join(result["Value"][1:3])): return subject = "%s: error executing %s" % (rb, name) else: return msg += " ".join(command) msg += "\nreturns: %s\n" % str(result["Value"][0]) + "\n".join(result["Value"][1:3]) msg += "\nUsing Proxy:\n" + getProxyInfoAsString(proxy)["Value"] # msg += '\nUsing Proxy:\n' + gProxyManager. ticketTime = self.errorClearTime + self.errorTicketTime if self.__ticketsWMSCache.exists(rb): mailAddress = self.alarmMailAddress # the RB was already detected failing a short time ago msg = ( "Submit GGUS Ticket for this error if not already opened\n" + "It has been failing at least for %s hours\n" % (ticketTime / 60 / 60) + msg ) else: self.__ticketsWMSCache.add(rb, ticketTime) if mailAddress: result = NotificationClient().sendMail(mailAddress, subject, msg, fromAddress=self.mailFromAddress) if not result["OK"]: self.log.error("Mail could not be sent") return
def __sendErrorMail( self, rb, name, command, result, proxy ): """ In case or error with RB/WM: - check if RB/WMS still in use - remove RB/WMS from current list - check if RB/WMS not in cache - add RB/WMS to cache - send Error mail """ if rb in self.resourceBrokers: try: self.resourceBrokers.remove( rb ) self.log.info( 'Removed RB from list', rb ) except: pass if not self.__failingWMSCache.exists( rb ): self.__failingWMSCache.add( rb, self.errorClearTime ) # disable for 30 minutes mailAddress = self.errorMailAddress msg = '' if not result['OK']: subject = "%s: timeout executing %s" % ( rb, name ) msg += '\n%s' % result['Message'] elif result['Value'][0] != 0: if re.search( 'the server is temporarily drained', ' '.join( result['Value'][1:3] ) ): return if re.search( 'System load is too high:', ' '.join( result['Value'][1:3] ) ): return subject = "%s: error executing %s" % ( rb, name ) else: return msg += ' '.join( command ) msg += '\nreturns: %s\n' % str( result['Value'][0] ) + '\n'.join( result['Value'][1:3] ) msg += '\nUsing Proxy:\n' + getProxyInfoAsString( proxy )['Value'] #msg += '\nUsing Proxy:\n' + gProxyManager. ticketTime = self.errorClearTime + self.errorTicketTime if self.__ticketsWMSCache.exists( rb ): mailAddress = self.alarmMailAddress # the RB was already detected failing a short time ago msg = 'Submit GGUS Ticket for this error if not already opened\n' + \ 'It has been failing at least for %s hours\n' % ( ticketTime / 60 / 60 ) + msg else: self.__ticketsWMSCache.add( rb, ticketTime ) if mailAddress: result = NotificationClient().sendMail( mailAddress, subject, msg, fromAddress = self.mailFromAddress ) if not result[ 'OK' ]: self.log.error( "Mail could not be sent" ) return
def __init__(self): ''' Constructor ''' super(ModuleBase, self).__init__() self.log = gLogger.getSubLogger("ModuleBase") result = getProxyInfoAsString() if not result['OK']: self.log.error( 'Could not obtain proxy information in module environment with message:\n', result['Message']) else: self.log.info('Payload proxy information:\n', result['Value']) self.ops = Operations() self.applicationLog = '' self.applicationVersion = '' self.applicationName = '' self.InputData = [] #Will have to become empty list self.SteeringFile = '' self.energy = 0 self.parametricParameters = "" #self.NumberOfEvents = 0 #self.WorkflowStartFrom = 0 self.result = S_ERROR() self.InputFile = [] self.ignoremissingInput = False self.OutputFile = '' self.jobType = '' self.stdError = '' self.debug = False self.extraCLIarguments = "" self.jobID = 0 if os.environ.has_key('JOBID'): self.jobID = os.environ['JOBID'] self.eventstring = [''] self.excludeAllButEventString = False self.ignoreapperrors = False self.inputdataMeta = {} ############# #Set from workflow object self.workflow_commons = {} self.step_commons = {} self.workflowStatus = S_OK() self.stepStatus = S_OK() self.isProdJob = False self.production_id = 0 self.prod_job_id = 0 self.jobName = "" self.request = None self.jobReport = None self.basedirectory = os.getcwd()
def __init__(self): ''' Constructor ''' super(ModuleBase, self).__init__() self.log = gLogger.getSubLogger("ModuleBase") result = getProxyInfoAsString() if not result['OK']: self.log.error('Could not obtain proxy information in module environment with message:\n', result['Message']) else: self.log.info('Payload proxy information:\n', result['Value']) self.ops = Operations() self.applicationLog = '' self.applicationVersion = '' self.applicationName = '' self.InputData = [] #Will have to become empty list self.SteeringFile = '' self.energy = 0 self.parametricParameters = "" #self.NumberOfEvents = 0 #self.WorkflowStartFrom = 0 self.result = S_ERROR() self.InputFile = [] self.ignoremissingInput = False self.OutputFile = '' self.jobType = '' self.stdError = '' self.debug = False self.extraCLIarguments = "" self.jobID = 0 if os.environ.has_key('JOBID'): self.jobID = os.environ['JOBID'] self.eventstring = [''] self.excludeAllButEventString = False self.ignoreapperrors = False self.inputdataMeta = {} ############# #Set from workflow object self.workflow_commons = {} self.step_commons = {} self.workflowStatus = S_OK() self.stepStatus = S_OK() self.isProdJob = False self.production_id = 0 self.prod_job_id = 0 self.jobName = "" self.request = None self.jobReport = None self.basedirectory = os.getcwd()
def __init__(self): """ Initialization of module base. """ super(ModuleBase, self).__init__() self.log = gLogger.getSubLogger('ModuleBase') # FIXME: do we need to do this for every module? result = getProxyInfoAsString() if not result['OK']: self.log.error('Could not obtain proxy information in module environment with message:\n', result['Message']) else: self.log.info('Payload proxy information:\n', result['Value']) self.ops = Operations() self.platform = '' self.applicationLog = '' self.applicationVersion = '' self.applicationName = '' self.InputData = [] #Will have to become empty list self.SteeringFile = '' self.energy = 0 self.NumberOfEvents = 0 self.WorkflowStartFrom = 0 self.result = S_ERROR() self.InputFile = [] self.ignoremissingInput = False self.OutputFile = '' self.jobType = '' self.stdError = '' self.debug = False self.extraCLIarguments = "" self.jobID = os.environ.get("JOBID", 0) self.eventstring = [''] self.excludeAllButEventString = False self.ignoreapperrors = False self.inputdataMeta = {} ############# #Set from workflow object self.workflow_commons = {} self.step_commons = {} self.workflowStatus = S_OK() self.stepStatus = S_OK() self.isProdJob = False self.productionID = 0 self.prod_job_id = 0 self.jobReport = None ## Make sure this is set everywhere os.environ['OMP_NUM_THREADS'] = '1'
def __init__(self): """ Initialization of module base. """ super(ModuleBase, self).__init__() self.log = gLogger.getSubLogger('ModuleBase') # FIXME: do we need to do this for every module? result = getProxyInfoAsString() if not result['OK']: self.log.error('Could not obtain proxy information in module environment with message:\n', result['Message']) else: self.log.info('Payload proxy information:\n', result['Value']) self.ops = Operations() self.platform = '' self.applicationLog = '' self.applicationVersion = '' self.applicationName = '' self.InputData = [] #Will have to become empty list self.SteeringFile = '' self.energy = 0 self.NumberOfEvents = 0 self.WorkflowStartFrom = 0 self.result = S_ERROR() self.InputFile = [] self.ignoremissingInput = False self.OutputFile = '' self.jobType = '' self.stdError = '' self.debug = False self.extraCLIarguments = "" self.jobID = os.environ.get("JOBID", 0) self.eventstring = [''] self.excludeAllButEventString = False self.ignoreapperrors = False self.inputdataMeta = {} ############# #Set from workflow object self.workflow_commons = {} self.step_commons = {} self.workflowStatus = S_OK() self.stepStatus = S_OK() self.isProdJob = False self.productionID = 0 self.prod_job_id = 0 self.jobReport = None ## Make sure this is set everywhere os.environ['OMP_NUM_THREADS'] = '1'
def writeProxyToFile(self, proxy): """CE helper function to write a CE proxy string to a file.""" result = writeToProxyFile(proxy) if not result["OK"]: self.log.error("Could not write proxy to file", result["Message"]) return result proxyLocation = result["Value"] result = getProxyInfoAsString(proxyLocation) if not result["OK"]: self.log.error("Could not get proxy info", result) return result else: self.log.info("Payload proxy information:") print(result["Value"]) return S_OK(proxyLocation)
def writeProxyToFile(self, proxy): """CE helper function to write a CE proxy string to a file. """ result = writeToProxyFile(proxy) if not result['OK']: self.log.error('Could not write proxy to file', result['Message']) return result proxyLocation = result['Value'] result = getProxyInfoAsString(proxyLocation) if not result['OK']: self.log.error('Could not get proxy info', result) return result else: self.log.info('Payload proxy information:') print result['Value'] return S_OK(proxyLocation)
def writeProxyToFile(self, proxy): """CE helper function to write a CE proxy string to a file. """ result = File.writeToProxyFile(proxy) if not result["OK"]: self.log.error("Could not write proxy to file", result["Message"]) return result proxyLocation = result["Value"] result = getProxyInfoAsString(proxyLocation) if not result["OK"]: self.log.error("Could not get proxy info", result) return result else: self.log.info("Payload proxy information:") print result["Value"] return S_OK(proxyLocation)
def writeProxyToFile( self, proxy ): """CE helper function to write a CE proxy string to a file. """ result = File.writeToProxyFile( proxy ) if not result[ 'OK' ]: self.log.error( 'Could not write proxy to file', result[ 'Message' ] ) return result proxyLocation = result[ 'Value' ] result = getProxyInfoAsString( proxyLocation ) if not result['OK']: self.log.error( 'Could not get proxy info', result ) return result else: self.log.info( 'Payload proxy information:' ) print result['Value'] return S_OK( proxyLocation )
def __init__(self): """ Initialization of module base. """ super(ModuleBase, self).__init__() self.log = gLogger.getSubLogger("ModuleBase") # FIXME: do we need to do this for every module? result = getProxyInfoAsString() if not result["OK"]: self.log.error( "Could not obtain proxy information in module environment with message:\n", result["Message"] ) else: self.log.info("Payload proxy information:\n", result["Value"]) self.ops = Operations() self.systemConfig = "" self.applicationLog = "" self.applicationVersion = "" self.applicationName = "" self.InputData = [] # Will have to become empty list self.SteeringFile = "" self.energy = 0 self.NumberOfEvents = 0 self.result = S_ERROR() self.InputFile = [] self.ignoremissingInput = False self.OutputFile = "" self.jobType = "" self.stdError = "" self.debug = False self.jobID = None if os.environ.has_key("JOBID"): self.jobID = os.environ["JOBID"] self.eventstring = [""] self.excludeAllButEventString = False self.ignoreapperrors = False self.inputdataMeta = {} ############# # Set from workflow object self.workflow_commons = {} self.step_commons = {} self.workflowStatus = S_OK() self.stepStatus = S_OK()