示例#1
0
文件: jobs.py 项目: exedre/e4t
def process_dataset(options,_dataset,processors,*args,**kw):
    """Given options, Information Set and processors filenames execute
   processors over the information set """
    for pr in processors:
        if not exists(pr):
            pr = join(options.process_path,pr)
            if not exists(pr):
                logger.error('E01#001: Processor config file not exists (%s)',pr)
                raise ValueError, 'E:MKEL:001'
        cfg = udict(cfg2hash(pr))

        if 'PROCESSOR' in cfg:
            pcfg = udict(cfg.xget('PROCESSOR'))            
            pname = pcfg.xget('NAME')
            if not pname:
                logger.error('E01#002: Key NAME not exists in processor config (%s)',pr)
                continue
            processor = getProcessor(pname,
                                     _dataset,
                                     config=pcfg)
            if processor:
                _dataset = processor.execute()
            else:
                logger.error('E01#003: Processor not exists (%s)',pname)
            
    return _dataset
示例#2
0
文件: __init__.py 项目: exedre/e4t
    def _inline_processor(self,_ts,name,kw):
        """ Inline processor """
        if kw['PROC']:
            # logger.debug( 'calling processor functions: %s on %s',kw['PROC'],name )
            names = (name,) if name else _ts.keys()
            for name in names:
                proc = getProcessor('INLINE_PROCESSOR',
                                    _ts,
                                    config=kw )
                _ts = proc.execute(name)  # returns full results 

        return _ts
示例#3
0
文件: __init__.py 项目: exedre/e4t
        else:
            if kw.has_key('NAME') and kw['NAME']:
                self._missing.extend(kw['NAME'].split(','))
            else:
                self._missing.extend([ r['Instrument'] for r in reqs])
            return 

        # ...if post-process 
        if kw['PROC']:
            # logger.debug('calling inline processor functions: %s',kw['PROC'])
            for name in _ts.keys():
                logger.debug('calling processor functions: %s for %s',kw['PROC'],name)
                # we re-take each time the processor to pass the Information Set
                # as modified from the previous operation
                proc = getProcessor('INLINE_PROCESSOR', 
                                    self._res , 
                                    config=kw )
                self._res = proc.execute(name)  # returns full results

        return self._res


class DatastreamProvider(WithSourceProvider):
    """
    Dataprovider facade for Thomson DataStream (DWE) source

    >>> dwe = DatastreamProvider()
    >>> dwe.append('dstream://Datastream/AGINDEMIF')
    >>> dwe.request()
    <DataSet of len    1:  AGINDEMIF(13)>
    """