def exec_job_COMPILE(options,*args,**kw):
"""Make .tex file for JOB"""
kw = udict(kw)
base = join(options.process_path,'input')
copy_file(kw,base)
datareqs, processors, outputs = e4t.cli.get_elem_configs(*args, **kw)
_dataset = None
elapsed = [.0,.0,.0]
if datareqs:
with Timer() as t:
_dataset = tsload_datareqs(options,_dataset,*datareqs,**kw)
logger.debug("TIME tsload %f",t.secs)
elapsed[0]=t.secs
if processors:
with Timer() as t:
_dataset = process_dataset(options,_dataset,processors,*args,**kw)
logger.debug("TIME process %f",t.secs)
elapsed[1]=t.secs
if not _dataset:
logger.error('VOID dataset on output step')
# raise IOError('VOID dataset')
if True: # outputs:
with Timer() as t:
output_dataset(options,_dataset,outputs,*args,**kw)
logger.debug("TIME output %f",t.secs)
elapsed[2]=t.secs
else:
logger.error('No conf for output step')
raise IOError('VOID output')
setattr(options,'elapsed',elapsed)
if options.switch_debug:
# Write the report file
e4t.cli.save_accounting("%s.cli" % options.job)
codecs.open("%s.time" % options.job,
'w',
'utf-8').write(u"tsload: %f\nprocss: %f\noutput: %f\n" % (elapsed[0],
elapsed[1],
elapsed[2]))
def __call__(self,options,*args,**kw):
"""Make .tex file for JOB"""
LOGGER.debug("compile")
kw = udict(kw)
base=join(options.process_path,'input')
copy_file(kw,base)
datareqs, processors, outputs = e4t.cli.get_elem_configs(*args, **kw)
_dataset = None
elapsed = [.0,.0,.0]
setattr(options,'elapsed',elapsed)
LOGGER.debug('## ------------------------------ LOAD ')
# Request phase
if datareqs:
with Timer() as t:
_dataset = load(*datareqs,options=options)
LOGGER.debug("TIME tsload %f",t.secs)
elapsed[0]=t.secs
setattr(options,'elapsed',elapsed)
LOGGER.debug('## ---------------------------- PROCESS ')
# Process pase
if processors:
with Timer() as t:
_dataset = process(_dataset,*processors,options=options)
LOGGER.debug("TIME process %f",t.secs)
elapsed[1]=t.secs
setattr(options,'elapsed',elapsed)
LOGGER.debug('## ---------------------------- OUTPUT ')
# Output phase
with Timer() as t:
from e4t.dataset.output import output
output(options,_dataset,*outputs,**kw)
LOGGER.debug("TIME output %f",t.secs)
elapsed[2]=t.secs
setattr(options,'elapsed',elapsed)
def exec_JOBS(options,*args,**kwargs):
logger.debug('JOBS OP ON %s - kw:%s',','.join(args),','.join([ "%s=%s" %(k,v) for k,v in kwargs.items()]))
ret,jlist,base,jobspecfile,order,jjspec = e4t.cli.select_jobs_from_jobfile(options.jobfile)
for jspec in jjspec:
copy_file(jspec,join(base,'input'))
for section,jobs in jlist.items():
if options.options.get_bool('section'):
if options.options.get_bool('label'):
print "section:",
print section
if 'exec' not in options.options:
for job in jobs:
if options.options.get_bool('labels'):
print "job:",
print job[0]
else:
execs = options.options['exec']
ret = []
for execf in execs:
acct.ok('JOB','BEGIN',execf)
r_step = execf
# pre phase function
try:
r=e4t.cli.exec_func('exec_job_'+execf.upper()+"_PRE",section,jobs,jobspecfile)
if r: ret.append(r)
except:
pass
for job in jobs:
r_job = job
setattr(options,'job',job[0].upper())
setattr(options,'switch_update',True)
if options.switch_verbose:
print '====== Execute JOB %s' % job[0].upper()
r = None
try:
r = e4t.cli.exec_func('exec_job_'+execf.upper(),job[0],**job[1])
acct.ok('JOB',execf,job[0])
except IOError, exc:
import traceback
import sys
stack = u''.join(traceback.format_tb(sys.exc_info()[2]))
stack = unicode(stack) if options.switch_debug else u''
msg=u"%10s | %s | T:%s \n %s\n" % (job[0].upper(),sys.exc_info()[1],
sys.exc_info()[0],
stack)
acct.ko('JOB',execf,job[0],dbg=msg)
if options.switch_debug:
logger.error(msg.encode('utf-8','ignore'))
else:
logger.error('Cannot execute %s',job[0])
if r: ret.append(r)
# if options.switch_verbose:
# print chkpnt.report(job,'DATA','PROC','OUTP','DONE',small=True)
# post-phase function
try:
r = e4t.cli.exec_func('exec_job_'+execf.upper()+"_POST",section,jobs,jobspecfile)
if r: ret.append(r)
except:
pass
acct.ok('JOB','END',execf)
# 'save' option
if 'save' in options.options:
save = options.options['save'][0]
logger.info('Save result in file %s',save)
open(save,'w').write('\n'.join(ret))
else: # print-out result
if ret:
print '\n'.join(ret)
def __call__(self,options,*args,**kwargs):
LOGGER.debug("calling JOBS command")
ERROR = Error()
setattr(options,'joblist',options.jobfile)
try:
ret,jlist,base,jobspecfile,order,jjspec = e4t.cli.select_jobs_from_jobfile(options.jobfile)
except Exception as exc:
ERROR.add(exc,options,'joblist','',sys.exc_info())
return ERROR
# Copy useful files from input directory (TODO: specify dir)
#
for jspec in jjspec:
copy_file(jspec,join(base,'input'))
for section,jobs in jlist.items():
setattr(options,'section',section.upper())
if options.options.get_bool('section'):
if options.options.get_bool('label'):
print "section:",
print section
if 'exec' not in options.options:
for job in jobs:
if options.options.get_bool('label'):
print "job:",
print job[0]
else:
execs = options.options['exec']
ret = []
for execf in execs:
setattr(options,'exec',section.upper())
ACCT.ok('PHASE','BEGIN',execf)
# pre phase function
try:
cmd = self.fo[execf]
except KeyError as exc:
ERROR.add(exc,options,'exec','',sys.exc_info())
raise E4tSystemError('EJ001: No phase %s',execf)
if not cmd:
LOGGER.error("Phase %s not in command",execf)
continue
COMMAND = cmd
setattr(options,'command_obj',self.fo)
try:
_err = COMMAND.setUp(section)
except Exception, exc:
ERROR.add(exc, options, 'exec','pre',sys.exc_info())
# Main jobs loop
for job in jobs:
setattr(options,'job',job[0].upper())
setattr(options,'switch_update',True)
if options.switch_verbose:
print('====== Execute JOB %s' % job[0].upper())
try:
_err = COMMAND(options,job[0],**job[1])
ACCT.ok('JOB',execf,job[0])
except Exception, exc:
ERROR.add(exc, options, 'job','mainloop',sys.exc_info())
ACCT.ko('JOB',execf,job[0])
# post-phase function
try:
_err = COMMAND.tearDown(section)
except Exception, exc:
ERROR.add(exc,options,'section','post',sys.exc_info())
ACCT.ok('PHASE','END',execf)