Example #1
0
File: make.py Project: exedre/e4t
def execute_joblist(joblist,options):
    """Execute a joblist
    

    :param joblist: the joblist specification
    :type joblist: a string, a url, a filename or a dict

    if the joblist is a URL then first download it from internet then execute it
    if the joblist is a dictionary it create a temp file and then execute it
    otherwise try to load the joblist from a file, eventually from the process_path directory and execute it
    delete the file if create (unless options.switch_debug)

    Options:
     * verbose

    Calls:
     * execute_job
    """
    fname = joblist
    f = None

    if is_url(joblist,DWNL_URL):
        f = download(joblist)
        fname = f.name

    if is_dict(joblist):
        f = True # has to unlink it
        fname = setup_job(joblist)
    elif not is_file(fname):
        if is_file(fname,options.process_path):
            fname = join(base,fname)
        else:
            logger.error('E:MAIN:JOB:001 -- Job %s spec not exists' % joblist)
            raise IOError('E:MAIN:JOB:001')

    # Esegui il lavoro
    if options.switch_verbose:
        print '==================== Execute JOBSPEC %s'% fname
        #report.info('==================== Execute JOBSPEC %s',fname)
    execute_job(fname)
Example #2
0
def _element2url(e,options):
    """Complete URL in string 

    .. todo:: Provider plugin must have a proper option
    """
    _url_completion = {
        'r': lambda x: "random://%s" % x,
        'fr': lambda x: "fred://fred/%s" % x,
        'ds': lambda x: "dstream://Datastream/%s" % x,
        'f': lambda x:  "flinp://DB22/%s" % x,
        'b': lambda x: "biss://DB22/%s" % x
        }
    if is_file(e) or is_url(e):
        return e
    f = _url_completion[options.behavior](e) 
    if not re.match('^ *#',e) and len(e.strip())>0:
        LOGGER.debug('URL COMPLETION |%s|=|%s|',e,f)
        return f
    return e
Example #3
0
def select_jobs_from_jobfile(job):
    """
    :param job: an url, a dict or a file-name of the jobfile

    :returns: a tuple containing 
     1. return code (0 is ok, otherwise fail)
     2. the jobs dict
     3. the base processing directory
     4. the whole jobfile 
     5. the order of the jobs to run
     6. the list of jobs sections 
    """
    logger.debug('JOBFILE=%s',job)

    fname = job
    f = None
    jjobs=udict()
    base = None
    job_specfile = None

    if is_url(job,DWNL_URL):
        f = download(job)
        fname = f.name
    if is_dict(job):
        f = True
        fname = setup_job(job)
    elif not is_file(fname):
        base = options.process_path
        logger.debug('BASE=%s',base)
        if is_file(fname,base):
            fname = join(base,fname)
        else:
            logger.error('E:MAIN:JOB:001 -- job spec file %s not exists (%s)', job,fname)
            raise IOError('E:MAIN:JOB:001')

    # Esegui il lavoro
    if options.switch_verbose:
        print '==================== Execute JOBFILE %s' % fname

    (ret, base, job_specfile) = _read_specfile(fname)
    if ret!=0:
        return ret

    if not options.process_path:
        options.process_path = dirname(fname)

    macros = job_specfile.xget('macros')
    if macros:
        macros = job_specfile['macros']
        logger.debug('Found MACROS in jobsfile (%s)',macros)

    # Select sections to operate on
    names = options.name

    _a = udict()
    
    if names is None:
        # If ha no name select BASE section
        if 'BASE' in job_specfile:
            basespec = job_specfile.xget_dict('BASE')
            if 'JOBS' in basespec:
                names = ('BASE',)
                # logger.debug('Take Jobs from BASE (%s)',','.join(names))
                _a['cli.jobs']='base'
        elif 'JOB' in job_specfile:
            # logger.debug('Make synthetic joblist for JOB')
            _a['cli.jobs']='synth'
            job_specfile['BASE']=udict(JOBS='JOB',kind='joblist')
            names=('BASE',)
        elif options.only:
            # logger.debug('Make synthetic joblist for ONLY jobs')
            _a['cli.jobs']='synth-only'
            job_specfile['BASE']=udict(JOBS=','.join(options.only),kind='joblist')
            names=('BASE',)

    if isinstance(names,basestring):
        names =  names.split(',')

    if not names:
        logging.error('No specfile joblist present in job spec file')
        return
    
    _a['cli.jobs']=','.join(names)
     
    # logger.debug('Sections to operate on %s',','.join(names))

    _accounting.update(_a)
    
    jjobs,order,jjspec = select_jobs_from_joblist(names,job_specfile,options,macros)

    _accounting['cli.order']=','.join(order)

    if f:
        if not options.switch_debug:
            os.unlink(fname)
            _accounting['fname']='deleted'

    return 0,jjobs,base,job_specfile,order,jjspec