def execute_joblist(joblist,options):
"""Execute a joblist
:param joblist: the joblist specification
:type joblist: a string, a url, a filename or a dict
if the joblist is a URL then first download it from internet then execute it
if the joblist is a dictionary it create a temp file and then execute it
otherwise try to load the joblist from a file, eventually from the process_path directory and execute it
delete the file if create (unless options.switch_debug)
Options:
* verbose
Calls:
* execute_job
"""
fname = joblist
f = None
if is_url(joblist,DWNL_URL):
f = download(joblist)
fname = f.name
if is_dict(joblist):
f = True # has to unlink it
fname = setup_job(joblist)
elif not is_file(fname):
if is_file(fname,options.process_path):
fname = join(base,fname)
else:
logger.error('E:MAIN:JOB:001 -- Job %s spec not exists' % joblist)
raise IOError('E:MAIN:JOB:001')
# Esegui il lavoro
if options.switch_verbose:
print '==================== Execute JOBSPEC %s'% fname
#report.info('==================== Execute JOBSPEC %s',fname)
execute_job(fname)
def _element2url(e,options):
"""Complete URL in string
.. todo:: Provider plugin must have a proper option
"""
_url_completion = {
'r': lambda x: "random://%s" % x,
'fr': lambda x: "fred://fred/%s" % x,
'ds': lambda x: "dstream://Datastream/%s" % x,
'f': lambda x: "flinp://DB22/%s" % x,
'b': lambda x: "biss://DB22/%s" % x
}
if is_file(e) or is_url(e):
return e
f = _url_completion[options.behavior](e)
if not re.match('^ *#',e) and len(e.strip())>0:
LOGGER.debug('URL COMPLETION |%s|=|%s|',e,f)
return f
return e
def select_jobs_from_jobfile(job):
"""
:param job: an url, a dict or a file-name of the jobfile
:returns: a tuple containing
1. return code (0 is ok, otherwise fail)
2. the jobs dict
3. the base processing directory
4. the whole jobfile
5. the order of the jobs to run
6. the list of jobs sections
"""
logger.debug('JOBFILE=%s',job)
fname = job
f = None
jjobs=udict()
base = None
job_specfile = None
if is_url(job,DWNL_URL):
f = download(job)
fname = f.name
if is_dict(job):
f = True
fname = setup_job(job)
elif not is_file(fname):
base = options.process_path
logger.debug('BASE=%s',base)
if is_file(fname,base):
fname = join(base,fname)
else:
logger.error('E:MAIN:JOB:001 -- job spec file %s not exists (%s)', job,fname)
raise IOError('E:MAIN:JOB:001')
# Esegui il lavoro
if options.switch_verbose:
print '==================== Execute JOBFILE %s' % fname
(ret, base, job_specfile) = _read_specfile(fname)
if ret!=0:
return ret
if not options.process_path:
options.process_path = dirname(fname)
macros = job_specfile.xget('macros')
if macros:
macros = job_specfile['macros']
logger.debug('Found MACROS in jobsfile (%s)',macros)
# Select sections to operate on
names = options.name
_a = udict()
if names is None:
# If ha no name select BASE section
if 'BASE' in job_specfile:
basespec = job_specfile.xget_dict('BASE')
if 'JOBS' in basespec:
names = ('BASE',)
# logger.debug('Take Jobs from BASE (%s)',','.join(names))
_a['cli.jobs']='base'
elif 'JOB' in job_specfile:
# logger.debug('Make synthetic joblist for JOB')
_a['cli.jobs']='synth'
job_specfile['BASE']=udict(JOBS='JOB',kind='joblist')
names=('BASE',)
elif options.only:
# logger.debug('Make synthetic joblist for ONLY jobs')
_a['cli.jobs']='synth-only'
job_specfile['BASE']=udict(JOBS=','.join(options.only),kind='joblist')
names=('BASE',)
if isinstance(names,basestring):
names = names.split(',')
if not names:
logging.error('No specfile joblist present in job spec file')
return
_a['cli.jobs']=','.join(names)
# logger.debug('Sections to operate on %s',','.join(names))
_accounting.update(_a)
jjobs,order,jjspec = select_jobs_from_joblist(names,job_specfile,options,macros)
_accounting['cli.order']=','.join(order)
if f:
if not options.switch_debug:
os.unlink(fname)
_accounting['fname']='deleted'
return 0,jjobs,base,job_specfile,order,jjspec