# private data members

__runjob = "RunJobHopper" # String defining the sub class

__instance = None # Boolean used by subclasses to become a Singleton

#__error = PilotErrors() # PilotErrors object

# Required methods

def __init__(self):

""" Default initialization """

# e.g. self.__errorLabel = errorLabel

pass

def __new__(cls, *args, **kwargs):

""" Override the __new__ method to make the class a singleton """

if not cls.__instance:

cls.__instance = super(RunJobHPC, cls).__new__(cls, *args, **kwargs)

return cls.__instance

def getRunJob(self):

""" Return a string with the execution class name """

return self.__runjob

def getRunJobFileName(self):

""" Return the filename of the module """

return super(RunJobHopper, self).getRunJobFileName()

# def argumentParser(self): <-- see example in RunJob.py

def allowLoopingJobKiller(self):

""" Should the pilot search for looping jobs? """

# The pilot has the ability to monitor the payload work directory. If there are no updated files within a certain

# time limit, the pilot will consider the as stuck (looping) and will kill it. The looping time limits are set

# in environment.py (see e.g. loopingLimitDefaultProd)

return False

def get_backfill(self, partition, max_nodes = None):

# Function collect information about current available resources and

# return number of nodes with possible maximum value for walltime according Titan policy

#

cmd = 'showbf -p %s' % partition

res_tuple = commands.getstatusoutput(cmd)

showbf_str = ""

if res_tuple[0] == 0:

showbf_str = res_tuple[1]

res = {}

tolog ("Available resources in %s partition" % partition)

tolog (showbf_str)

if showbf_str:

shobf_out = showbf_str.splitlines()

tolog ("Fitted resources")

for l in shobf_out[2:]:

d = l.split()

nodes = int(d[2])

if not d[3] == 'INFINITY':

wal_time_arr = d[3].split(":")

if len(wal_time_arr) < 4:

wal_time_sec = int(wal_time_arr[0])*(60*60) + int(wal_time_arr[1])*60 + int(wal_time_arr[2])

if wal_time_sec > 24 * 3600:

wal_time_sec = 24 * 3600

else:

wal_time_sec = 24 * 3600

if nodes > 1:

nodes = nodes - 1

else:

wal_time_sec = 12 * 3600

# Fitting Hopper policy

# https://www.nersc.gov/users/computational-systems/hopper/running-jobs/queues-and-policies/

nodes = max_nodes if nodes > max_nodes else nodes

if nodes < 65 and wal_time_sec > 96 * 3600:

wal_time_sec = 96 * 3600

elif nodes < 683 and wal_time_sec > 48 * 3600:

wal_time_sec = 48 * 3600

elif nodes < 4097 and wal_time_sec > 36 * 3600:

wal_time_sec = 36 * 3600

elif wal_time_sec > 12 * 3600:

wal_time_sec = 12 * 3600

tolog ("Nodes: %s, Walltime (str): %s, Walltime (min) %s" % (nodes, d[3], wal_time_sec/60 ))

res.update({nodes:wal_time_sec})

else:

tolog ("No availble resources. Default values will be used.")

return res

def get_hpc_resources(self, partition, max_nodes = None, min_nodes = 1, min_walltime = 30):

#

# Function return number of nodes and walltime for submission

#

nodes = min_nodes

walltime = min_walltime

backfill = self.get_backfill(partition, max_nodes)

if backfill:

for n in sorted(backfill.keys(), reverse=True):

if min_walltime <= backfill[n] and nodes <= n:

nodes = n

walltime = backfill[n] / 60

walltime = walltime - 2

break

if walltime <= 0:

walltime = min_walltime

nodes = min_nodes

return nodes, walltime

def jobStateChangeNotification(self, src_obj, fire_on, value):

tolog("Job state changed to '%s'" % value)

return True

def executePayload(self, thisExperiment, runCommandList, job, repeat_num = 0):

""" execute the payload """

t0 = os.times()

res_tuple = (0, 'Undefined')

# special setup command. should be placed in queue defenition (or job defenition) ?

setup_commands = ['source $MODULESHOME/init/bash',

'export LD_LIBRARY_PATH=/opt/cray/lib64:$LD_LIBRARY_PATH',

'export CRAY_ROOTFS=DSL',

'module load python']

# loop over all run commands (only >1 for multi-trfs)

current_job_number = 0

getstatusoutput_was_interrupted = False

number_of_jobs = len(runCommandList)

for cmd in runCommandList:

nodes, walltime = self.get_hpc_resources(self.partition_comp, self.max_nodes, self.nodes, self.min_walltime)

cpu_number = self.cpu_number_per_node * nodes

tolog("Launch parameters \nWalltime limit : %s (min)\nRequested nodes (cores): %s (%s)" % (walltime,nodes,cpu_number))

current_job_number += 1

try:

# add the full job command to the job_setup.sh file

to_script = "\n".join(cmd['environment'])

to_script = to_script + ("\nexport G4FORCENUMBEROFTHREADS=%s" % self.number_of_threads) # needed for GEANT

to_script = to_script + "\n" + "\n".join(setup_commands)

to_script = "%s\naprun -n %s -d %s %s %s" % (to_script, cpu_number/self.number_of_threads, self.number_of_threads ,cmd["payload"], cmd["parameters"])

thisExperiment.updateJobSetupScript(job.workdir, to_script=to_script)

# Simple SAGA fork variant

tolog("******* SAGA call to execute payload *********")

try:

js = saga.job.Service("pbs://localhost")

jd = saga.job.Description()

if self.project_id:

jd.project = self.project_id # should be taken from resourse description (pandaqueue)

jd.wall_time_limit = walltime

jd.executable = to_script

jd.total_cpu_count = cpu_number

jd.output = job.stdout

jd.error = job.stderr

jd.queue = self.executed_queue # should be taken from resourse description (pandaqueue)

jd.working_directory = job.workdir

fork_job = js.create_job(jd)

fork_job.add_callback(saga.STATE, self.jobStateChangeNotification)

#tolog("\n(PBS) Command: %s\n" % to_script)

fork_job.run()

#tolog("\nCommand was started at %s.\nState is: %s" % ( str(datetime.now()), fork_job.state))

tolog("Local Job ID: %s" % fork_job.id)

for i in range(self.waittime * 60):

time.sleep(1)

if fork_job.state != saga.job.PENDING:

break

if fork_job.state == saga.job.PENDING:

repeat_num = repeat_num + 1

tolog("Wait time (%s minutes) exceed, job will be rescheduled (%s)" % (self.waittime, repeat_num))

fork_job.cancel()

fork_job.wait()

if repeat_num < 10:

return self.executePayload(thisExperiment, runCommandList, job, repeat_num)

fork_job.wait()

tolog("Job State : %s" % (fork_job.state))

tolog("Exitcode : %s" % (fork_job.exit_code))

tolog("Create time : %s" % (fork_job.created))

tolog("Start time : %s" % (fork_job.started))

tolog("End time : %s" % (fork_job.finished))

tolog("Walltime limit : %s (min)" % (jd.wall_time_limit))

tolog("Allocated nodes (cores): %s (%s)" % (nodes,cpu_number))

cons_time = datetime.strptime(fork_job.finished, '%c') - datetime.strptime(fork_job.started, '%c')

cons_time_sec = (cons_time.microseconds + (cons_time.seconds + cons_time.days * 24 * 3600) * 10**6) / 10**6

tolog("Execution time : %s (sec. %s)" % (str(cons_time), cons_time_sec))

#job.timeExe = int(fork_job.finished - fork_job.started)

res_tuple = (fork_job.exit_code, "Look into: %s" % job.stdout)

####################################################

except saga.SagaException, ex:

# Catch all saga exceptions

tolog("An exception occured: (%s) %s " % (ex.type, (str(ex))))

# Trace back the exception. That can be helpful for debugging.

tolog(" \n*** Backtrace:\n %s" % ex.traceback)

break

tolog("**********************************************")

tolog("******* SAGA call finished *******************")

tolog("**********************************************")

except Exception, e:

tolog("!!FAILED!!3000!! Failed to run command %s" % str(e))

getstatusoutput_was_interrupted = True

if self.getFailureCode():

job.result[2] = self.getFailureCode()

tolog("!!FAILED!!3000!! Failure code: %d" % (self.getFailureCode()))

break

if res_tuple[0] == 0:

tolog("Job command %d/%d finished" % (current_job_number, number_of_jobs))

else:

tolog("Job command %d/%d failed: res = %s" % (current_job_number, number_of_jobs, str(res_tuple)))

break

t1 = os.times()

t = map(lambda x, y:x-y, t1, t0) # get the time consumed

job.cpuConsumptionUnit, job.cpuConsumptionTime, job.cpuConversionFactor = pUtil.setTimeConsumed(t)

tolog("Job CPU usage: %s %s" % (job.cpuConsumptionTime, job.cpuConsumptionUnit))

tolog("Job CPU conversion factor: %1.10f" % (job.cpuConversionFactor))

job.timeExe = int(round(t1[4] - t0[4]))

tolog("Original exit code: %s" % (res_tuple[0]))

if res_tuple[0] != None:

tolog("Exit code: %s (returned from OS)" % (res_tuple[0]%255))

res0, exitAcronym, exitMsg = self.getTrfExitInfo(res_tuple[0], job.workdir)

else:

tolog("Exit code: None (returned from OS, Job was canceled)")

res0 = None

exitMsg = "Job was canceled by internal call"

# check the job report for any exit code that should replace the res_tuple[0]

res = (res0, res_tuple[1], exitMsg)

# dump an extract of the payload output

if number_of_jobs > 1:

_stdout = job.stdout

_stderr = job.stderr

_stdout = _stdout.replace(".txt", "_N.txt")

_stderr = _stderr.replace(".txt", "_N.txt")

tolog("NOTE: For %s output, see files %s, %s (N = [1, %d])" % (job.payload, _stdout, _stderr, number_of_jobs))

else:

tolog("NOTE: For %s output, see files %s, %s" % (job.payload, job.stdout, job.stderr))

# JEM job-end callback

try:

from JEMstub import notifyJobEnd2JEM

notifyJobEnd2JEM(job, tolog)

except:

pass # don't care (fire and forget)

""" signal handler """

error = PilotErrors()

runJob.setGlobalPilotErrorDiag("!!FAILED!!3000!! SIGTERM Signal %s is caught in child pid=%d!\n" % (sig, os.getpid()))

tolog(runJob.getGlobalPilotErrorDiag())

if sig == signal.SIGTERM:

runJob.setGlobalErrorCode(error.ERR_SIGTERM)

elif sig == signal.SIGQUIT:

runJob.setGlobalErrorCode(error.ERR_SIGQUIT)

elif sig == signal.SIGSEGV:

runJob.setGlobalErrorCode(error.ERR_SIGSEGV)

elif sig == signal.SIGXCPU:

runJob.setGlobalErrorCode(error.ERR_SIGXCPU)

elif sig == signal.SIGBUS:

runJob.setGlobalErrorCode(error.ERR_SIGBUS)

elif sig == signal.SIGUSR1:

runJob.setGlobalErrorCode(error.ERR_SIGUSR1)

else:

runJob.setGlobalErrorCode(error.ERR_KILLSIGNAL)

runJob.setFailureCode(runJob.getGlobalErrorCode)

# print to stderr

print >> sys.stderr, runJob.getGlobalPilotErrorDiag()