def setupWorkflow( self, debug = False ): """ Setup job workflow by defining the sequence of all executables All parameters shall have been defined before that method is called. """ iStep=1 #### execute setup software: needed for next steps swStep = self.setExecutable( 'cta-prod3-setupsw', arguments='%s %s'% (self.package, self.version),\ logFile='SetupSoftware_Log.txt') swStep['Value']['name'] = 'Step%i_SetupSoftware' % iStep swStep['Value']['descr_short'] = 'Setup software' iStep+=1 ### execute corsika step if self.runType == 'corsika': csStep = self.setExecutable( './cta-prod3-corsika.py', arguments="%s"% (self.input_card),\ ## optionally one can add a prefix to be used to build the output file name #arguments="%s %s"% (self.input_card, 'outprefix' ),\ logFile='Corsika_Log.txt') csStep['Value']['name'] = 'Step%i_Corsika' % iStep csStep['Value']['descr_short'] = 'Run corsika' iStep+=1 #### execute simtel_array step if self.runType == 'simtel': simStep = self.setExecutable( './dirac_prod3_simtel_only', arguments='%s %s'% (self.simtelcfg, self.simtelopts),\ logFile='Simtel_Log.txt') simStep['Value']['name'] = 'Step%i_Simtel' % iStep simStep['Value']['descr_short'] = 'Run sim_telarray' iStep+=1 # execute read_cta step if self.runType == 'readcta': res = sourceEnv(600, ['prod3_types'], {} ) read_cta_opts=res['outputEnv']['read_cta_opts'] rctaStep = self.setExecutable( './dirac_prod3_read_cta', \ arguments = "-q -r 4 -u --min-trg-tel 2 %s" % (read_cta_opts), logFile = 'ReadCta_Log.txt' ) rctaStep['Value']['name'] = 'Step%i_ReadCta' % iStep rctaStep['Value']['descr_short'] = 'Run ReadCta' iStep += 1
def executeGridCommand( proxy, cmd, gridEnvScript = None ): """ Execute cmd tuple after sourcing GridEnv """ currentEnv = dict( os.environ ) if not gridEnvScript: # if not passed as argument, use default from CS Helpers gridEnvScript = Local.gridEnv() if gridEnvScript: command = gridEnvScript.split() ret = sourceEnv( 10, command ) if not ret['OK']: return S_ERROR( 'Failed sourcing GridEnv: %s' % ret['Message'] ) gridEnv = ret['outputEnv'] # # Preserve some current settings if they are there # if currentEnv.has_key( 'X509_VOMS_DIR' ): gridEnv['X509_VOMS_DIR'] = currentEnv['X509_VOMS_DIR'] if currentEnv.has_key( 'X509_CERT_DIR' ): gridEnv['X509_CERT_DIR'] = currentEnv['X509_CERT_DIR'] else: gridEnv = currentEnv if not proxy: res = getProxyInfo() if not res['OK']: return res gridEnv['X509_USER_PROXY' ] = res['Value']['path'] elif type( proxy ) in types.StringTypes: if os.path.exists( proxy ): gridEnv[ 'X509_USER_PROXY' ] = proxy else: return S_ERROR( 'Can not treat proxy passed as a string' ) else: ret = gProxyManager.dumpProxyToFile( proxy ) if not ret['OK']: return ret gridEnv[ 'X509_USER_PROXY' ] = ret['Value'] result = systemCall( 120, cmd, env = gridEnv ) return result
def executeGridCommand(proxy, cmd, gridEnvScript=None): """ Execute cmd tuple after sourcing GridEnv """ currentEnv = dict(os.environ) if not gridEnvScript: # if not passed as argument, use default from CS Helpers gridEnvScript = Local.gridEnv() if gridEnvScript: command = gridEnvScript.split() ret = sourceEnv(10, command) if not ret['OK']: return S_ERROR('Failed sourcing GridEnv: %s' % ret['Message']) gridEnv = ret['outputEnv'] # # Preserve some current settings if they are there # if currentEnv.has_key('X509_VOMS_DIR'): gridEnv['X509_VOMS_DIR'] = currentEnv['X509_VOMS_DIR'] if currentEnv.has_key('X509_CERT_DIR'): gridEnv['X509_CERT_DIR'] = currentEnv['X509_CERT_DIR'] else: gridEnv = currentEnv if not proxy: res = getProxyInfo() if not res['OK']: return res gridEnv['X509_USER_PROXY'] = res['Value']['path'] elif type(proxy) in types.StringTypes: if os.path.exists(proxy): gridEnv['X509_USER_PROXY'] = proxy else: return S_ERROR('Can not treat proxy passed as a string') else: ret = gProxyManager.dumpProxyToFile(proxy) if not ret['OK']: return ret gridEnv['X509_USER_PROXY'] = ret['Value'] result = systemCall(120, cmd, env=gridEnv) return result
def executeGridCommand(proxy, cmd, gridEnvScript=None): """ Execute cmd tuple after sourcing GridEnv """ currentEnv = dict(os.environ) if not gridEnvScript: # if not passed as argument, use default from CS Helpers gridEnvScript = Local.gridEnv() if gridEnvScript: command = gridEnvScript.split() ret = sourceEnv(10, command) if not ret["OK"]: return S_ERROR("Failed sourcing GridEnv: %s" % ret["Message"]) gridEnv = ret["outputEnv"] # # Preserve some current settings if they are there # if "X509_VOMS_DIR" in currentEnv: gridEnv["X509_VOMS_DIR"] = currentEnv["X509_VOMS_DIR"] if "X509_CERT_DIR" in currentEnv: gridEnv["X509_CERT_DIR"] = currentEnv["X509_CERT_DIR"] else: gridEnv = currentEnv if not proxy: res = getProxyInfo() if not res["OK"]: return res gridEnv["X509_USER_PROXY"] = res["Value"]["path"] elif isinstance(proxy, str): if os.path.exists(proxy): gridEnv["X509_USER_PROXY"] = proxy else: return S_ERROR("Can not treat proxy passed as a string") else: ret = gProxyManager.dumpProxyToFile(proxy) if not ret["OK"]: return ret gridEnv["X509_USER_PROXY"] = ret["Value"] result = systemCall(120, cmd, env=gridEnv) return result
def executeGridCommand( proxy, cmd, gridEnvScript = None ): """ Execute cmd tuple after sourcing GridEnv """ currentEnv = dict( os.environ ) if gridEnvScript: ret = sourceEnv( 10, [gridEnvScript] ) if not ret['OK']: return S_ERROR( 'Failed sourcing GridEnv: %s' % ret['Message'] ) gridEnv = ret['outputEnv'] # # Preserve some current settings if they are there # if currentEnv.has_key( 'X509_VOMS_DIR' ): gridEnv['X509_VOMS_DIR'] = currentEnv['X509_VOMS_DIR'] if currentEnv.has_key( 'X509_CERT_DIR' ): gridEnv['X509_CERT_DIR'] = currentEnv['X509_CERT_DIR'] else: gridEnv = currentEnv dumpedProxy = False if not proxy: res = getProxyInfo() if not res['OK']: return res gridEnv['X509_USER_PROXY' ] = res['Value']['path'] elif type( proxy ) in types.StringTypes: if os.path.exists( proxy ): gridEnv[ 'X509_USER_PROXY' ] = proxy else: return S_ERROR( 'Can not treat proxy passed as a string' ) else: ret = gProxyManager.dumpProxyToFile( proxy ) if not ret['OK']: return ret gridEnv[ 'X509_USER_PROXY' ] = ret['Value'] dumpedProxy = True result = systemCall( 120, cmd, env = gridEnv ) if dumpedProxy: # If proxy is stored in a file, delete it, gProxyManager fails to reuse it. # Once the gProxyManager is fixed, this file removal can be dropped os.unlink( gridEnv[ 'X509_USER_PROXY' ] ) return result
def executeGridCommand(proxy, cmd, gridEnvScript=None): """ Execute cmd tuple after sourcing GridEnv """ currentEnv = dict(os.environ) if gridEnvScript: ret = sourceEnv(10, [gridEnvScript]) if not ret["OK"]: return S_ERROR("Failed sourcing GridEnv: %s" % ret["Message"]) gridEnv = ret["outputEnv"] # # Preserve some current settings if they are there # if currentEnv.has_key("X509_VOMS_DIR"): gridEnv["X509_VOMS_DIR"] = currentEnv["X509_VOMS_DIR"] if currentEnv.has_key("X509_CERT_DIR"): gridEnv["X509_CERT_DIR"] = currentEnv["X509_CERT_DIR"] else: gridEnv = currentEnv if not proxy: res = getProxyInfo() if not res["OK"]: return res gridEnv["X509_USER_PROXY"] = res["Value"]["path"] elif type(proxy) in types.StringTypes: if os.path.exists(proxy): gridEnv["X509_USER_PROXY"] = proxy else: return S_ERROR("Can not treat proxy passed as a string") else: ret = gProxyManager.dumpProxyToFile(proxy) if not ret["OK"]: return ret gridEnv["X509_USER_PROXY"] = ret["Value"] result = systemCall(120, cmd, env=gridEnv) return result
def __init__( self ): """ Standard constructor """ self.log = gLogger.getSubLogger( 'LSFTimeLeft' ) self.jobID = os.environ.get( 'LSB_JOBID' ) self.queue = os.environ.get( 'LSB_QUEUE' ) self.bin = os.environ.get( 'LSF_BINDIR' ) self.host = os.environ.get( 'LSB_HOSTS' ) self.year = time.strftime( '%Y', time.gmtime() ) self.log.verbose( 'LSB_JOBID=%s, LSB_QUEUE=%s, LSF_BINDIR=%s, LSB_HOSTS=%s' % ( self.jobID, self.queue, self.bin, self.host ) ) self.cpuLimit = None self.cpuRef = None self.normRef = None self.wallClockLimit = None self.hostNorm = None cmd = '%s/bqueues -l %s' % ( self.bin, self.queue ) result = runCommand( cmd ) if not result['OK']: return self.log.debug( 'From %s' % cmd, result['Value'] ) lines = str( result['Value'] ).split( '\n' ) for i in xrange( len( lines ) ): if re.search( '.*CPULIMIT.*', lines[i] ): info = lines[i + 1].split() if len( info ) >= 4: self.cpuLimit = float( info[0] ) * 60 self.cpuRef = info[3] else: self.log.warn( 'Problem parsing "%s" for CPU limit' % lines[i + 1] ) self.cpuLimit = -1 if re.search( '.*RUNLIMIT.*', lines[i] ): info = lines[i + 1].split() if len( info ) >= 1: self.wallClockLimit = float( info[0] ) * 60 else: self.log.warn( 'Problem parsing "%s" for wall clock limit' % lines[i + 1] ) modelMaxNorm = 0 if self.cpuRef: # Now try to get the CPU_FACTOR for this reference CPU, # it must be either a Model, a Host or the largest Model cmd = '%s/lshosts -w %s' % ( self.bin, self.cpuRef ) result = runCommand( cmd ) if result['OK']: # At CERN this command will return an error since there is no host defined # with the name of the reference Host. lines = str( result['Value'] ).split( '\n' ) l1 = lines[0].split() l2 = lines[1].split() if len( l1 ) > len( l2 ): self.log.error( "Failed lshost command", "%s:\n %s\n %s" % ( cmd, lines[0], lines[0] ) ) else: for i in range( len( l1 ) ): if l1[i] == 'cpuf': try: self.normRef = float( l2[i] ) self.log.info( 'Reference Normalization taken from Host', '%s: %s' % ( self.cpuRef, self.normRef ) ) except ValueError as e: self.log.exception( 'Exception parsing lshosts output', '', e ) if not self.normRef: # Try if there is a model define with the name of cpuRef cmd = '%s/lsinfo -m' % ( self.bin ) result = runCommand( cmd ) if result['OK']: lines = str( result['Value'] ).split( '\n' ) for line in lines[1:]: words = line.split() if len( words ) > 1: try: norm = float( words[1] ) if norm > modelMaxNorm: modelMaxNorm = norm if words[0].find( self.cpuRef ) > -1: self.normRef = norm self.log.info( 'Reference Normalization taken from Host Model', '%s: %s' % ( self.cpuRef, self.normRef ) ) except ValueError as e: self.log.exception( 'Exception parsing lsfinfo output', '', e ) if not self.normRef: # Now parse LSF configuration files if not os.path.isfile( './lsf.sh' ): os.symlink( os.path.join( os.environ['LSF_ENVDIR'], 'lsf.conf' ) , './lsf.sh' ) # As the variables are not exported, we must force it ret = sourceEnv( 10, ['./lsf', '&& export LSF_CONFDIR' ] ) if ret['OK']: lsfEnv = ret['outputEnv'] shared = None try: egoShared = os.path.join( lsfEnv['LSF_CONFDIR'], 'ego.shared' ) lsfShared = os.path.join( lsfEnv['LSF_CONFDIR'], 'lsf.shared' ) if os.path.exists( egoShared ): shared = egoShared elif os.path.exists( lsfShared ): shared = lsfShared except KeyError as e: self.log.exception( 'Exception getting LSF configuration', '', e ) if shared: f = open( shared ) hostModelSection = False for line in f.readlines(): if line.find( 'Begin HostModel' ) == 0: hostModelSection = True continue if not hostModelSection: continue if line.find( 'End HostModel' ) == 0: break line = line.strip() if line and line.split()[0] == self.cpuRef: try: self.normRef = float( line.split()[1] ) self.log.info( 'Reference Normalization taken from Configuration File', '(%s) %s: %s' % ( shared, self.cpuRef, self.normRef ) ) except ValueError as e: self.log.exception( 'Exception reading LSF configuration', '', e ) else: self.log.warn( 'Could not find LSF configuration' ) else: self.log.error( 'Cannot source the LSF environment', ret['Message'] ) if not self.normRef: # If nothing worked, take the maximum defined for a Model if modelMaxNorm: self.normRef = modelMaxNorm self.log.info( 'Reference Normalization taken from Max Model:', self.normRef ) # Now get the Normalization for the current Host if self.host: cmd = '%s/lshosts -w %s' % ( self.bin, self.host ) result = runCommand( cmd ) if result['OK']: lines = str( result['Value'] ).split( '\n' ) l1 = lines[0].split() l2 = lines[1].split() if len( l1 ) > len( l2 ): self.log.error( "Failed lshost command", "%s:\n %s\n %s" % ( cmd, lines[0], lines[0] ) ) else: for i in range( len( l1 ) ): if l1[i] == 'cpuf': try: self.hostNorm = float( l2[i] ) self.log.info( 'Host Normalization', '%s: %s' % ( self.host, self.hostNorm ) ) except ValueError as e: self.log.exception( 'Exception parsing lshosts output', '', e ) if self.hostNorm and self.normRef: self.hostNorm /= self.normRef self.log.info( 'CPU Normalization', self.hostNorm )
def __init__(self): """Standard constructor""" super(LSFResourceUsage, self).__init__("LSF", "LSB_JOBID") self.queue = os.environ.get("LSB_QUEUE") self.bin = os.environ.get("LSF_BINDIR") self.host = os.environ.get("LSB_HOSTS") self.year = time.strftime("%Y", time.gmtime()) self.log.verbose( "LSB_JOBID=%s, LSB_QUEUE=%s, LSF_BINDIR=%s, LSB_HOSTS=%s" % (self.jobID, self.queue, self.bin, self.host) ) self.cpuLimit = None self.cpuRef = None self.normRef = None self.wallClockLimit = None self.hostNorm = None cmd = "%s/bqueues -l %s" % (self.bin, self.queue) result = runCommand(cmd) if not result["OK"]: return lines = str(result["Value"]).split("\n") self.log.debug( "From %s" % cmd, "\n".join([line if len(line) <= 128 else line[:128] + " [...]" for line in lines]) ) for i in range(len(lines)): if re.search(".*CPULIMIT.*", lines[i]): info = lines[i + 1].split() if len(info) >= 4: self.cpuLimit = float(info[0]) * 60 self.cpuRef = info[3] elif len(info) == 2 and info[1] == "min": self.cpuLimit = float(info[0]) * 60 self.cpuRef = None else: self.log.warn('Problem parsing "%s" for CPU limit' % lines[i + 1]) self.cpuLimit = -1 elif re.search(".*RUNLIMIT.*", lines[i]): info = lines[i + 1].split() if len(info) >= 1: self.wallClockLimit = float(info[0]) * 60 else: self.log.warn('Problem parsing "%s" for wall clock limit' % lines[i + 1]) self.wallClockLimit = -1 modelMaxNorm = 0 if self.cpuRef: # Now try to get the CPU_FACTOR for this reference CPU, # it must be either a Model, a Host or the largest Model cmd = "%s/lshosts -w %s" % (self.bin, self.cpuRef) result = runCommand(cmd) if result["OK"]: # At CERN this command will return an error since there is no host defined # with the name of the reference Host. lines = str(result["Value"]).split("\n") l1 = lines[0].split() l2 = lines[1].split() if len(l1) > len(l2): self.log.error("Failed lshost command", "%s:\n %s\n %s" % (cmd, lines[0], lines[0])) else: for i in range(len(l1)): if l1[i] == "cpuf": try: self.normRef = float(l2[i]) self.log.info( "Reference Normalization taken from Host", "%s: %s" % (self.cpuRef, self.normRef) ) except ValueError as e: self.log.exception("Exception parsing lshosts output", "", e) if not self.normRef: # Try if there is a model define with the name of cpuRef cmd = "%s/lsinfo -m" % (self.bin) result = runCommand(cmd) if result["OK"]: lines = str(result["Value"]).split("\n") for line in lines[1:]: words = line.split() if len(words) > 1: try: norm = float(words[1]) if norm > modelMaxNorm: modelMaxNorm = norm if words[0].find(self.cpuRef) > -1: self.normRef = norm self.log.info( "Reference Normalization taken from Host Model", "%s: %s" % (self.cpuRef, self.normRef), ) except ValueError as e: self.log.exception("Exception parsing lsfinfo output", "", e) if not self.normRef: # Now parse LSF configuration files if not os.path.isfile("./lsf.sh"): os.symlink(os.path.join(os.environ["LSF_ENVDIR"], "lsf.conf"), "./lsf.sh") # As the variables are not exported, we must force it ret = sourceEnv(10, ["./lsf", "&& export LSF_CONFDIR"]) if ret["OK"]: lsfEnv = ret["outputEnv"] shared = None try: egoShared = os.path.join(lsfEnv["LSF_CONFDIR"], "ego.shared") lsfShared = os.path.join(lsfEnv["LSF_CONFDIR"], "lsf.shared") if os.path.exists(egoShared): shared = egoShared elif os.path.exists(lsfShared): shared = lsfShared except KeyError as e: self.log.exception("Exception getting LSF configuration", "", e) if shared: with open(shared) as f: hostModelSection = False for line in f.readlines(): if line.find("Begin HostModel") == 0: hostModelSection = True continue if not hostModelSection: continue if line.find("End HostModel") == 0: break line = line.strip() if line and line.split()[0] == self.cpuRef: try: self.normRef = float(line.split()[1]) self.log.info( "Reference Normalization taken from Configuration File", "(%s) %s: %s" % (shared, self.cpuRef, self.normRef), ) except ValueError as e: self.log.exception("Exception reading LSF configuration", "", e) else: self.log.warn("Could not find LSF configuration") else: self.log.error("Cannot source the LSF environment", ret["Message"]) if not self.normRef: # If nothing works take this as the unit self.normRef = 1.0 # If nothing worked, take the maximum defined for a Model # if modelMaxNorm: # self.normRef = modelMaxNorm # self.log.info('Reference Normalization taken from Max Model:', self.normRef) # Now get the Normalization for the current Host if self.host: cmd = "%s/lshosts -w %s" % (self.bin, self.host) result = runCommand(cmd) if result["OK"]: lines = str(result["Value"]).split("\n") l1 = lines[0].split() l2 = lines[1].split() if len(l1) > len(l2): self.log.error("Failed lshost command", "%s:\n %s\n %s" % (cmd, lines[0], lines[0])) else: for i in range(len(l1)): if l1[i] == "cpuf": try: self.hostNorm = float(l2[i]) self.log.info("Host Normalization", "%s: %s" % (self.host, self.hostNorm)) except ValueError as e: self.log.exception("Exception parsing lshosts output", l1, e) finally: break if self.hostNorm and self.normRef: self.hostNorm /= self.normRef self.log.info("CPU power w.r.t. batch unit", self.hostNorm) if self.hostNorm: # Set the limits in real seconds self.cpuLimit /= self.hostNorm self.wallClockLimit /= self.hostNorm
def __init__(self): """ Standard constructor """ self.log = gLogger.getSubLogger('LSFTimeLeft') self.jobID = os.environ.get('LSB_JOBID') self.queue = os.environ.get('LSB_QUEUE') self.bin = os.environ.get('LSF_BINDIR') self.host = os.environ.get('LSB_HOSTS') self.year = time.strftime('%Y', time.gmtime()) self.log.verbose( 'LSB_JOBID=%s, LSB_QUEUE=%s, LSF_BINDIR=%s, LSB_HOSTS=%s' % (self.jobID, self.queue, self.bin, self.host)) self.cpuLimit = None self.cpuRef = None self.normRef = None self.wallClockLimit = None self.hostNorm = None cmd = '%s/bqueues -l %s' % (self.bin, self.queue) result = runCommand(cmd) if not result['OK']: return lines = str(result['Value']).split('\n') self.log.debug( 'From %s' % cmd, '\n'.join([ line if len(line) <= 128 else line[:128] + ' [...]' for line in lines ])) for i in xrange(len(lines)): if re.search('.*CPULIMIT.*', lines[i]): info = lines[i + 1].split() if len(info) >= 4: self.cpuLimit = float(info[0]) * 60 self.cpuRef = info[3] elif len(info) == 2 and info[1] == "min": self.cpuLimit = float(info[0]) * 60 self.cpuRef = None else: self.log.warn('Problem parsing "%s" for CPU limit' % lines[i + 1]) self.cpuLimit = -1 elif re.search('.*RUNLIMIT.*', lines[i]): info = lines[i + 1].split() if len(info) >= 1: self.wallClockLimit = float(info[0]) * 60 else: self.log.warn('Problem parsing "%s" for wall clock limit' % lines[i + 1]) self.wallClockLimit = -1 modelMaxNorm = 0 if self.cpuRef: # Now try to get the CPU_FACTOR for this reference CPU, # it must be either a Model, a Host or the largest Model cmd = '%s/lshosts -w %s' % (self.bin, self.cpuRef) result = runCommand(cmd) if result['OK']: # At CERN this command will return an error since there is no host defined # with the name of the reference Host. lines = str(result['Value']).split('\n') l1 = lines[0].split() l2 = lines[1].split() if len(l1) > len(l2): self.log.error("Failed lshost command", "%s:\n %s\n %s" % (cmd, lines[0], lines[0])) else: for i in xrange(len(l1)): if l1[i] == 'cpuf': try: self.normRef = float(l2[i]) self.log.info( 'Reference Normalization taken from Host', '%s: %s' % (self.cpuRef, self.normRef)) except ValueError as e: self.log.exception( 'Exception parsing lshosts output', '', e) if not self.normRef: # Try if there is a model define with the name of cpuRef cmd = '%s/lsinfo -m' % (self.bin) result = runCommand(cmd) if result['OK']: lines = str(result['Value']).split('\n') for line in lines[1:]: words = line.split() if len(words) > 1: try: norm = float(words[1]) if norm > modelMaxNorm: modelMaxNorm = norm if words[0].find(self.cpuRef) > -1: self.normRef = norm self.log.info( 'Reference Normalization taken from Host Model', '%s: %s' % (self.cpuRef, self.normRef)) except ValueError as e: self.log.exception( 'Exception parsing lsfinfo output', '', e) if not self.normRef: # Now parse LSF configuration files if not os.path.isfile('./lsf.sh'): os.symlink( os.path.join(os.environ['LSF_ENVDIR'], 'lsf.conf'), './lsf.sh') # As the variables are not exported, we must force it ret = sourceEnv(10, ['./lsf', '&& export LSF_CONFDIR']) if ret['OK']: lsfEnv = ret['outputEnv'] shared = None try: egoShared = os.path.join(lsfEnv['LSF_CONFDIR'], 'ego.shared') lsfShared = os.path.join(lsfEnv['LSF_CONFDIR'], 'lsf.shared') if os.path.exists(egoShared): shared = egoShared elif os.path.exists(lsfShared): shared = lsfShared except KeyError as e: self.log.exception( 'Exception getting LSF configuration', '', e) if shared: with open(shared) as f: hostModelSection = False for line in f.readlines(): if line.find('Begin HostModel') == 0: hostModelSection = True continue if not hostModelSection: continue if line.find('End HostModel') == 0: break line = line.strip() if line and line.split()[0] == self.cpuRef: try: self.normRef = float(line.split()[1]) self.log.info( 'Reference Normalization taken from Configuration File', '(%s) %s: %s' % (shared, self.cpuRef, self.normRef)) except ValueError as e: self.log.exception( 'Exception reading LSF configuration', '', e) else: self.log.warn('Could not find LSF configuration') else: self.log.error('Cannot source the LSF environment', ret['Message']) if not self.normRef: # If nothing works take this as the unit self.normRef = 1. # If nothing worked, take the maximum defined for a Model # if modelMaxNorm: # self.normRef = modelMaxNorm # self.log.info( 'Reference Normalization taken from Max Model:', self.normRef ) # Now get the Normalization for the current Host if self.host: cmd = '%s/lshosts -w %s' % (self.bin, self.host) result = runCommand(cmd) if result['OK']: lines = str(result['Value']).split('\n') l1 = lines[0].split() l2 = lines[1].split() if len(l1) > len(l2): self.log.error("Failed lshost command", "%s:\n %s\n %s" % (cmd, lines[0], lines[0])) else: for i in xrange(len(l1)): if l1[i] == 'cpuf': try: self.hostNorm = float(l2[i]) self.log.info( 'Host Normalization', '%s: %s' % (self.host, self.hostNorm)) except ValueError as e: self.log.exception( 'Exception parsing lshosts output', l1, e) finally: break if self.hostNorm and self.normRef: self.hostNorm /= self.normRef self.log.info('CPU power w.r.t. batch unit', self.hostNorm) if self.hostNorm: # Set the limits in real seconds self.cpuLimit /= self.hostNorm self.wallClockLimit /= self.hostNorm