# private data members

__runjob = "RunJobArgo" # String defining the sub class

__instance = None # Boolean used by subclasses to become a Singleton

__error = PilotErrors() # PilotErrors object

# public data members

process = ""# zjet, wjet, wqq, wcjet, etc.

base_filename = "alpout" # should be the same as in the input cards

# controls for warmup

warmup_phase0_number_events = None

warmup_phase0_number_iterations = None

warmup_phase1_number_events = None

warmup_wall_minutes = None

warmup_preprocess = 'alpgen_warmup_presubmit.sh'

warmup_preprocess_args = None

# controls for event generation (weighted gen + unweighting)

evtgen_phase0_number_events = None

evtgen_phase0_number_iterations = None

evtgen_phase1_number_events = None

evtgen_nodes = None

evtgen_processes_per_node = None

evtgen_wall_minutes = None

evtgen_executable = 'alpgenCombo.sh'

evtgen_scheduler_args = '--mode=script'

evtgen_preprocess = 'alpgen_presubmit.sh'

evtgen_postprocess = 'alpgen_postsubmit.sh'

working_path = None

input_url = None

output_url = None

pdf_filename = 'cteq6l1.tbl'

username = None

serial_site = 'argo_cluster'

parallel_site = None

group_identifier = None

athena_input_card_executable = 'get_alpgen_input_card.py'

athena_postprocess = 'alpgen_create_input_cards.py'

athena_postprocess_log = 'alpgen_create_input_cards.log'

ecm = None

run_number = None

job_config = None

evgen_job_opts = None

athena_input_card_name = 'input_card.mode_1.dat' # card output by Generate_trf

grid_ftp_server = 'atlasgridftp02.hep.anl.gov'

grid_ftp_protocol = 'gsiftp://'

job_working_path = '/grid/atlas/hpc/argo/jobs'

argo_job = []

# Required methods

def __init__(self):

""" Default initialization """

pass

def __new__(cls, *args, **kwargs):

""" Override the __new__ method to make the class a singleton """

if not cls.__instance:

cls.__instance = super(RunJobHPC, cls).__new__(cls, *args, **kwargs)

return cls.__instance

def getRunJob(self):

""" Return a string with the experiment name """

return self.__runjob

def getRunJobFileName(self):

""" Return the filename of the module """

return super(RunJobArgo, self).getRunJobFileName()

# def argumentParser(self): <-- see example in RunJob.py

def allowLoopingJobKiller(self):

""" Should the pilot search for looping jobs? """

# The pilot has the ability to monitor the payload work directory. If there are no updated files within a certain

# time limit, the pilot will consider the as stuck (looping) and will kill it. The looping time limits are set

# in environment.py (see e.g. loopingLimitDefaultProd)

return False

def get_argo_job(self, job):

##-----------------------

# create argo job

##-----------------------

argo_job = ArgoJob()

argo_job.input_url = None #self.GRID_FTP_PROTOCOL + self.GRID_FTP_SERVER + self.job_path

if self.input_url is not None:

argo_job.input_url = self.input_url

argo_job.output_url = self.grid_ftp_protocol + self.grid_ftp_server + self.job_path

if self.output_url is not None:

argo_job.output_url = self.output_url

argo_job.username = self.username

argo_job.group_identifier = self.group_identifier

##-----------------------

# create get alpgen input cards balsam job

##-----------------------

input_file_imode0 = self.base_filename + '.input.0'

input_file_imode1 = self.base_filename + '.input.1'

input_file_imode2 = self.base_filename + '.input.2'

input_cards_job = BalsamJob()

input_cards_job.executable = self.athena_input_card_executable

input_cards_job.executable_args = ('-e ' + self.ecm

+ ' -r ' + self.run_number

+ ' -o ' + self.job_config

+ ' -j ' + self.evgen_job_opts)

input_cards_job.output_files = [input_file_imode0,

input_file_imode1,

input_file_imode2,

self.athena_postprocess_log]

input_cards_job.nodes = 1

input_cards_job.processes_per_node = 1

input_cards_job.wall_minutes = 0 # running on condor cluster so does not need time

input_cards_job.username = self.username

input_cards_job.target_site = self.serial_site

input_cards_job.postprocess = self.athena_postprocess

input_cards_job.postprocess_args = (' -i ' + self.athena_input_card_name + ' -p ' + self.process

+ ' -n ' + str(self.evtgen_phase1_number_events)

+ ' --log-filename=' + str(self.athena_postprocess_log))

if self.warmup_phase0_number_events is not None:

input_cards_job.postprocess_args += ' --wmp-evts-itr=' + str(self.warmup_phase0_number_events)

if self.warmup_phase0_number_iterations is not None:

input_cards_job.postprocess_args += ' --wmp-nitr=' + str(self.warmup_phase0_number_iterations)

if self.warmup_phase1_number_events is not None:

input_cards_job.postprocess_args += ' --wmp-evts=' + str(self.warmup_phase1_number_events)

argo_job.add_job(input_cards_job)

##-----------------------

# create warm-up job

##-----------------------

# create grid filenames

grid1 = self.base_filename + '.grid1'

grid2 = self.base_filename + '.grid2'

# create warmup balsam job

warmup = BalsamJob()

warmup.executable = self.process + 'gen90_mpi'

warmup.executable_args = input_file_imode0

warmup.input_files = [input_file_imode0]

warmup.output_files = [grid1,grid2]

warmup.nodes = 1

warmup.processes_per_node = 1

warmup.wall_minutes = 0 # running on condor cluster so does not need time

warmup.username = self.username

warmup.target_site = self.serial_site

warmup.preprocess = self.warmup_preprocess

argo_job.add_job(warmup)

##-----------------------

# create event generation job

##-----------------------

# create executable

alpgen_exe = self.process + 'gen90_mpi_ramdisk_nomrstpdfs'

if 'argo_cluster' in self.parallel_site: # no ramdisk needed on argo_cluster

alpgen_exe = self.process + 'gen90_mpi'

# create filenames

unw = self.base_filename + '.unw.gz'

unw_par = self.base_filename + '_unw.par'

wgt = self.base_filename + '.wgt'

wgt_par = self.base_filename + '.par'

directoryList_before = 'directoryList_before.txt'

directoryList_after = 'directoryList_after.txt'

# create event gen balsam job

evtgen = BalsamJob()

evtgen.executable = self.evtgen_executable

evtgen.executable_args = (alpgen_exe + ' ' + input_file_imode1 + ' '

+ input_file_imode2 + ' ' + str(self.evtgen_processes_per_node))

evtgen.input_files = [grid1,

grid2,

input_file_imode1,

input_file_imode2]

evtgen.output_files = [unw,

unw_par,

directoryList_before,

directoryList_after,

self.evtgen_postprocess + '.out',

self.evtgen_postprocess + '.err',

]

evtgen.preprocess = self.evtgen_preprocess

evtgen.postprocess = self.evtgen_postprocess

evtgen.postprocess_args = self.base_filename

evtgen.nodes = self.evtgen_nodes

evtgen.processes_per_node = self.evtgen_processes_per_node

evtgen.wall_minutes = self.evtgen_wall_minutes

evtgen.username = self.username

evtgen.scheduler_args = self.evtgen_scheduler_args

evtgen.target_site = self.parallel_site

argo_job.add_job(evtgen)

return argo_job

def setup(self, job, jobSite, thisExperiment):

""" prepare the setup and get the run command list """

# start setup time counter

t0 = time.time()

ec = 0

# split up the job parameters to be able to loop over the tasks

jobParameters = job.jobPars.split("\n")[0]

jobTrf = job.trf.split("\n")[0]

parser = optparse.OptionParser(description=' program to submit alpgen jobs like a pilot')

parser.add_option('-p','--process',dest='process',help='Alpgen Process, i.e. zjet, wjet, wqq, etc.')

parser.add_option('-n','--nevts',dest='nevts',help='Number of weighted events requested in input file for weighted event generation',type='int')

parser.add_option('-g','--group-id',dest='group_identifier',help='User specified string that helps the user group jobs together.')

parser.add_option('-e','--ecm',dest='ecm',help='Center of Mass Energy.')

parser.add_option('-r','--run-number',dest='run_number',help='Run Number')

parser.add_option('-c','--jobConfig',dest='jobConfig',help='Job Options that will used from the Job Config tarball, i.e. MC12JobOptions/MC12.<Run Number>.<description>.py')

parser.add_option('-j','--evgenJobOpts',dest='evgenJobOpts',help='Job Config tarball, i.e. MC12JobOpts-XX-YY-ZZ.tar.gz')

parser.add_option('','--dev',dest='dev',help='For development only.',action='store_true',default=False)

parser.add_option('-q','--status-queue',dest='enable_status_queue',help='Enable the setting of the message queue parameter in the ArgoJob, which means ARGO will not send message updates for this job to the queue with its job ID.',action='store_true',default=False)

#parser.add_option('-a','--warmup-evts',dest='warmup_evts',help='For Warmup Step: Three numbers seperated by commas giving the number of events per iteration, number of iterations, and final number of events to generate. Example: "10000,10,1000000"')

parser.add_option('-b','--evtgen-evts',dest='evtgen_evts',help='For Event Generation Step: The number of events to generation in the event generation step. The ouput of unweighted events tends to be less so request more than you want. For example W+0jets gives you 70\%, W+1jet gives you 16%, W+2jet gives you 5%, W+3jet gives you 1%, and so on.', type='int')

parser.add_option('-o','--num-nodes',dest='numnodes',help='number of nodes to use on destination machine',type='int')

parser.add_option('-u','--ranks-per-node',dest='ranks_per_node',help='number of MPI ranks per node to use on destination machine',type='int')

parser.add_option('-t','--wall-time',dest='walltime',help='The wall time to submit to the queue in minutes.',type='int')

parser.add_option('-s','--site',dest='site',help='Balsam site name on which to run the event generation')

parser.add_option('-x','--no-submit',dest='submit',help='do not submit the message to ARGO. For testing purposes.',action='store_false',default=True)

parser.add_option('','--wmp-evts-itr',dest='wm_evts_per_itr',help='Warmup: Number of weighted events per interation.')

parser.add_option('','--wmp-nitr',dest='wm_nitr',help='Warmup: Number of iterations')

parser.add_option('','--wmp-evts',dest='wm_evts',help='Warmup: Number of final events to produce.')

try:

options, args = parser.parse_args(shlex.split(jobParameters))

except:

ec = self.__error.ERR_SETUPFAILURE

job.pilotErrorDiag = "Failure to parse job arguments"

tolog("Failure to parse job arguments for ARGO job")

return ec, job

tolog("ARGO job will be launched with next parameters: %s" % jobParameters)

self.process = options.process

self.username = 'pilot, %s' % job.prodUserID[:120] #os.environ['USER']

self.group_identifier = options.group_identifier

self.ecm = options.ecm

self.run_number = options.run_number

self.job_config = options.jobConfig

self.evgen_job_opts = options.evgenJobOpts

self.warmup_phase0_number_events = options.wm_evts_per_itr

self.warmup_phase0_number_iterations = options.wm_nitr

self.warmup_phase1_number_events = options.wm_evts

self.evtgen_phase1_number_events = options.evtgen_evts

self.evtgen_nodes = options.numnodes

self.evtgen_processes_per_node = options.ranks_per_node

self.evtgen_wall_minutes = options.walltime

self.parallel_site = options.site

self.dev = options.dev

self.job_path = os.path.join(self.job_working_path,job.jobId)

tolog("ARGO job path: %s" % self.job_path)

self.argo_job = self.get_argo_job(job)

if options.dev:

job.serial_site = 'argo_cluster_dev'

# verify that the multi-trf job is setup properly

os.chdir(jobSite.workdir)

tolog("Current job workdir is %s" % os.getcwd())

job.timeSetup = int(time.time() - t0)

tolog("Total setup time: %d s" % (job.timeSetup))

return ec, job

def executePayload(self, thisExperiment, job):

t0 = os.times()

res_tuple = None

# loop over all run commands (only >1 for multi-trfs)

getstatusoutput_was_interrupted = False

job_status = None

tolog("About to launch ARGO job")

# Poll MQ for Job Status

try:

# Initiate MQ interface and send job

self.argo_job.job_status_routing_key = '%s_job_status' % job.jobId #'status_' + jobID

si = SiteInformation()

mi = MessageInterface()

mi.host = 'atlasgridftp02.hep.anl.gov'

mi.port = 5671

mi.ssl_cert = si.getSSLCertificate() #'/grid/atlas/hpc/pilot_certs/xrootdsrv-cert.pem'

proxy_cert_path = si.getSSLCertificate()

mi.ssl_cert = os.path.dirname(proxy_cert_path) + "/rabbitmq-cert.pem"

if 'X509_USER_CERT' in os.environ.keys():

mi.ssl_cert = os.environ['X509_USER_CERT'] #'/users/hpcusers/balsam_dev/gridsecurity/jchilders/xrootdsrv-cert.pem'

mi.ssl_key = mi.ssl_cert #'/grid/atlas/hpc/pilot_certs/xrootdsrv-key.pem'

mi.ssl_key = os.path.dirname(proxy_cert_path) + "/rabbitmq-key.pem"

if 'X509_USER_KEY' in os.environ.keys():

mi.ssl_key = os.environ['X509_USER_KEY'] #'/users/hpcusers/balsam_dev/gridsecurity/jchilders/xrootdsrv-key.pem'

#mi.ssl_ca_certs = os.path.dirname(proxy_cert_path) + "/rabbitmq-cacerts.pem"

mi.ssl_ca_certs = '/grid/atlas/hpc/pilot_certs/cacerts.pem'

#if 'X509_CA_CERTS' in os.environ.keys():

# mi.ssl_ca_certs = os.environ['X509_CA_CERTS'] #'/users/hpcusers/balsam_dev/gridsecurity/jchilders/cacerts.pem'

#tolog("CA certs: %s" % (mi.ssl_ca_certs))

ca_certs = os.path.dirname(proxy_cert_path) + "/rabbitmq-cacerts.pem"

if os.path.isfile(ca_certs):

mi.ssl_ca_certs = ca_certs

mi.exchange_name = 'argo_users'

#Create queue to get messages about ARGO Job status from MQ

tolog('Opening connection with MQ')

mi.open_blocking_connection()

tolog('Create queue [%s] to retrieve messages with job status' % self.argo_job.job_status_routing_key)

mi.create_queue(self.argo_job.job_status_routing_key, self.argo_job.job_status_routing_key)

# submit ARGO job to MQ

#tolog('Opening connection with MQ')

#mi.open_blocking_connection()

routing_key = 'argo_job'

if self.dev:

routing_key = 'argo_job_dev'

tolog('Sending msg with job to ARGO')

mi.send_msg(self.argo_job.serialize(), routing_key)

tolog(' done sending ')

# Waiting till job done or failed

ARGO_err_msg = ''

while True:

time.sleep(5)

message = mi.receive_msg(self.argo_job.job_status_routing_key, True)

if message[2]:

tolog ("Got message from queue [%s]: method [%s], properties [%s], body [ %s ]" % (self.argo_job.job_status_routing_key, message[0], message[1], message[2]))

job_status = ArgoJobStatus.get_from_message(message[2])

job.hpcStatus = job_status.state

rt = RunJobUtilities.updatePilotServer(job, self.getPilotServer(), self.getPilotPort())

tolog("Extracted state: %s" % job_status.state)

if job_status.state == job_status.HISTORY:

res_tuple = (0, "Done")

break

elif job_status.is_failed():

res_tuple = (1, "Failed")

ARGO_err_msg = ARGO_err_msg + ' ' + job_status.message

elif job_status.state == job_status.FAILED:

res_tuple = (1, "Failed")

ARGO_err_msg = ARGO_err_msg + ' ' + job_status.message

runJob.failJob(1, 0, job, ins=job.inFiles, pilotErrorDiag=ARGO_err_msg)

break

time.sleep(5)

mi.close()

tolog(' closing connection to MQ')

tolog("Job State: %s" % (job_status.state))

#job.timeExe = int(fork_job.finished - fork_job.started)

####################################################

except Exception, e:

tolog("!!FAILED!!3000!! Failed to run command %s" % str(e))

getstatusoutput_was_interrupted = True

res_tuple = (1, "Failed")

self.failJob(0, self.__error.ERR_GENERALERROR, job, pilotErrorDiag=str(e))

else:

if res_tuple[0] == 0:

tolog("ARGO Job finished")

else:

tolog("ARGO Job failed: res = %s" % (str(res_tuple)))

t1 = os.times()

# CPU consumption metrics

# t = map(lambda x, y:x-y, t1, t0) # get the time consumed

# job.cpuConsumptionUnit, job.cpuConsumptionTime, job.cpuConversionFactor = pUtil.setTimeConsumed(t)

# tolog("Job CPU usage: %s %s" % (job.cpuConsumptionTime, job.cpuConsumptionUnit))

# tolog("Job CPU conversion factor: %1.10f" % (job.cpuConversionFactor))

job.timeExe = int(round(t1[4] - t0[4]))

tolog("Original exit code: %s" % (res_tuple[0]))

if res_tuple[0] != None:

tolog("Exit code: %s (returned from OS)" % (res_tuple[0]%255))

res0 = res_tuple[0]%255

if job_status:

exitMsg = job_status.message

else:

exitMsg = res_tuple[1]

else:

tolog("Exit code: None (returned from OS, Job was canceled or interrupted)")

res0 = None

exitMsg = "Job was canceled by internal call"

# check the job report for any exit code that should replace the res_tuple[0]

res = (res0, res_tuple[1], exitMsg)

# dump an extract of the payload output

tolog("NOTE: For %s output, see files %s, %s" % (job.payload, job.stdout, job.stderr))

# JEM job-end callback

try:

from JEMstub import notifyJobEnd2JEM

notifyJobEnd2JEM(job, tolog)

except:

pass # don't care (fire and forget)

""" signal handler """

error = PilotErrors()

runJob.setGlobalPilotErrorDiag("!!FAILED!!3000!! SIGTERM Signal %s is caught in child pid=%d!\n" % (sig, os.getpid()))

tolog(runJob.getGlobalPilotErrorDiag())

if sig == signal.SIGTERM:

runJob.setGlobalErrorCode(error.ERR_SIGTERM)

elif sig == signal.SIGQUIT:

runJob.setGlobalErrorCode(error.ERR_SIGQUIT)

elif sig == signal.SIGSEGV:

runJob.setGlobalErrorCode(error.ERR_SIGSEGV)

elif sig == signal.SIGXCPU:

runJob.setGlobalErrorCode(error.ERR_SIGXCPU)

elif sig == signal.SIGBUS:

runJob.setGlobalErrorCode(error.ERR_SIGBUS)

elif sig == signal.SIGUSR1:

runJob.setGlobalErrorCode(error.ERR_SIGUSR1)

else:

runJob.setGlobalErrorCode(error.ERR_KILLSIGNAL)

runJob.setFailureCode(runJob.getGlobalErrorCode)

# print to stderr

print >> sys.stderr, runJob.getGlobalPilotErrorDiag()