def save_definition(options,formats,_dataset,outputs,_cfgs,extn,*args,**kw):
kw=udict(kw)
label = kw.xget('LABEL').upper()
kind = kw.xget('KIND').upper()
parts = udict({ 'page' : LaTeXPage,
'addtotoc': LaTeXAddToToc,
'part' : LaTeXPart,
'chapter' : LaTeXChapter,
'toc' : LaTeXTOC,
'cover' : LaTeXCover,
'empty' : LaTeXEmpty,
'table' : OutputTable,
'figure' : OutputFigure,
'dataset' : None,
'multiple' : OutputMultiple
})
if kind not in parts.keys():
raise ValueError, "Document part %s not found" % kind
elem_class = parts[kind]
fname = "%s.tex" % options.job
defn = []
for ou in outputs:
if not exists(ou):
ou = join(options.process_path,ou)
if not exists(ou):
logger.error('E01#001: Il file di specifica del processore %s non esiste',pr)
raise ValueError, 'E:MKEL:001'
wcfg = udict(cfg2hash(ou))
lcfg = wcfg.xget(label)
if exists(ou):
spec = cfg2hash(ou)
kw=udict(kw)
if 'macros' in kw:
spec['macros']=kw['macros']
else:
logger.error('{OUTPUTELEMENT}:E:INIT:001 -- Il file di configurazione %s non esiste', ou)
raise ValueError, "E:OUTPUTELEMENT:INIT:001"
spec = mapspec(label,spec,_dataset,options,
lambda k,v,name,spec,_dataset,options:
(k, expandfuncs(v,name,spec,_dataset,options)))
t = elem_class(label,spec,_dataset,options)
xml = t.to_xml()
defn.append( xml )
xmls = dom.parseString(xml) # or xml.dom.minidom.parseString(xml_string)
pp = Postprocessor(options=options)
pp.output(False,xmls.toprettyxml(),'%s.xml'% options.job)
def writeout_processor_file(elem,options=Options()):
"""Write Processor File
:param elem: the processor specification
:type elem: a filename or specification line
:Example:
e4t --request random://uniform/VAR~:2012-12-31~2012-01-01~D \
--processor FORMULA=J;J=VAR*100
"""
if not elem:
return
_dict = udict()
for i,e in enumerate(elem):
if exists(e):
_dict = cfg2hash(e)
else:
_d = udict({ 'kind':'processor',
'active':'yes' ,
'name': 'BASIC',
'operations': 'FORMULA',
})
_in=as_dict(e,";")
_d.update(_in)
if not _d.has_key('FORMULA'):
_d['FORMULA']=','.join(sorted([ k for k in _in.keys() if k not in ('DROP',) ]))
if len(_d['FORMULA'])==0: del _d['FORMULA']
job = 'PROC%d'%i
_dict[job]=_d
_dict['PROCESSOR']=udict({ 'processors': ','.join(sorted(_dict.keys()))})
return writeout_file(_dict,header='# processor file')
def process_dataset(options,_dataset,processors,*args,**kw):
"""Given options, Information Set and processors filenames execute
processors over the information set """
for pr in processors:
if not exists(pr):
pr = join(options.process_path,pr)
if not exists(pr):
logger.error('E01#001: Processor config file not exists (%s)',pr)
raise ValueError, 'E:MKEL:001'
cfg = udict(cfg2hash(pr))
if 'PROCESSOR' in cfg:
pcfg = udict(cfg.xget('PROCESSOR'))
pname = pcfg.xget('NAME')
if not pname:
logger.error('E01#002: Key NAME not exists in processor config (%s)',pr)
continue
processor = getProcessor(pname,
_dataset,
config=pcfg)
if processor:
_dataset = processor.execute()
else:
logger.error('E01#003: Processor not exists (%s)',pname)
return _dataset
def get_elem_configs(*args,**kw):
datareqs,processors,outputs = [],[],[]
kw = udict(kw)
base = dirname(kw['__FILE'])
# ... to acquire data
d = kw.xget('DATAREQ')
if d:
pd = _i(d,base,options.process_path)
if pd:
datareqs.append(pd)
else:
logger.warn('Data Request file "%s" not exists',d)
# ..to process 'em
d = kw.xget('PROCESSORS')
if d:
pd = _i(d,base,options.process_path)
if pd:
processors.append(pd)
else:
logger.warn('Processor Conf file "%s" not exists',d)
# Job Label
# Use section name if not exists LABEL key
l = kw.xget('LABEL',args[0].upper())
#
_os = kw.xget_list('OUTPUT')
if _os:
for d in _os:
pd = _i(d,base,options.process_path)
if pd:
outputs.append(pd)
else:
logger.warn('Output Conf file "%s" not exists',d)
for o in outputs:
cfg = cfg2hash(o)
jcfg = cfg.xget(l.upper())
if jcfg:
base = dirname(jcfg.xget('__FILE'))
d = jcfg.xget('DATAREQ')
pd = _i(d,base,options.process_path)
if pd:
datareqs.append(pd)
else:
logger.warn('*Data Request file "%s" not exists',d)
d = jcfg.xget('PROCESSOR')
if d:
pd = _i(d,base,options.process_path)
if pd:
processors.append(pd)
else:
logger.warn('*Processor Conf file "%s" not exists',d)
return datareqs,processors,outputs
def _setup(self,label,config):
if config:
cfg = cfg2hash(config)
if cfg.has_key(label):
self.update(cfg[label])
self._global = cfg
self[u'E4T_WORK']=label
def output_dataset(options,_dataset,outputs,*args,**kw):
kw = udict(kw)
label = kw.xget('LABEL').upper()
kind = kw.xget('KIND').upper()
parts = udict({ 'page' : LaTeXPage,
'addtotoc': LaTeXAddToToc,
'part' : LaTeXPart,
'chapter' : LaTeXChapter,
'toc' : LaTeXTOC,
'cover' : LaTeXCover,
'empty' : LaTeXEmpty,
'table' : OutputTable,
'figure' : OutputFigure,
'dataset' : None,
'multiple' : OutputMultiple})
if kind not in parts.keys():
raise ValueError, "Document part %s not found" % kind
fname = "%s.tex" % options.job
for ou in outputs:
if not exists(ou):
ou = join(options.process_path,ou)
if not exists(ou):
logger.error('E01#001: Il file di specifica del processore %s non esiste',pr)
raise ValueError, 'E:MKEL:001'
wcfg = udict(cfg2hash(ou))
lcfg = wcfg.xget(label)
with Timer() as t:
_output_element(options,label,ou,_dataset,parts[kind],fname,**kw)
_accounting["OUT.TIME.%s"%ou] = t.msecs
xfname = "%s.e4t" % options.job
logger.debug('Write out %s',xfname)
save_dict(xfname,_accounting,options.switch_debug)
_accounting.clear()
if 'OUTPUT' in kw:
return
# NON-OUTPUT ELEMENTS
if parts[kind] is None:
logger.error('Output error for %s',kind)
return
t = parts[kind](kw)
l = LaTeX(options)
if hasattr(t,'setup_elements'):
t.setup_elements()
if hasattr(t,'produce'):
t.produce()
ofile = fname
f = open(ofile, 'w')
f.write(l.layout(t,False))
f.close()
outfiles.append(ofile)
def _read_cfgfile(kw,fkey,label=None):
fn = udict(kw).xget(fkey)
if fn:
if not exists(fn):
fn = join(options.process_path,fn)
if not exists(fn):
logger.error('E01#001: cfg file for dataset %s missing',fn)
raise ValueError, 'E:MKEL:001'
cfg = udict(cfg2hash(fn))
if label:
cfg = udict(cfg).xget(label)
return cfg
def read_conf_file(cfg="~/.e4trc",options=Options(),section="E4T"):
"""Reads configuration file for resource into options before command line
:param cfg: configuration file name (defaults to ~/.srirc
:type cfg: file name
:param options: options to fill
:type options: Option class
.. todo:: move to utils
"""
opts = {
'process_path' : str,
'output_path': str,
'input_path' : str,
'pageno' : bool,
'components': bool,
'dataset_path' : str,
'dataset_url' : str ,
'dataset' : bool ,
'dataset_input' : bool,
'dataset_output' : bool,
'profile' : str,
'cache' : bool,
'verbose' : bool,
'debug' : bool,
'define' : str,
'logging' : str,
'jobfile' : str,
}
_d = {}
rc = expandvars(expanduser(cfg))
if exists(rc):
# read configuration file
_ch = cfg2hash(rc,klass=ldict)
# gets the section
if _ch.has_key(section):
# take only the keys in the opts list
_l = dict([ (k,v) for k,v in _ch[section].items() if k in opts.keys()])
# set the option (TODO: better treatment for option and management of non-switch options)
_l = [ ((("switch_%s" % k) if opts[k] is bool else k) ,
expandvars(expanduser(v)) if opts[k] is str else eval(v))
for k,v in _l.items() ]
# make it a dict
_d = dict(_l)
# definitions treatment
if _d.has_key('define'):
_d['define'] = _d['define'].split(',')
# now define the options in the Options() class
for k,v in _d.items():
setattr(options,k,v)
return options
def exec_job_TOC(options,*args,**kw):
kw = udict(kw)
kind = kw.xget('KIND')
if kind is None:
return
num = kw.xget('num','<?>')
title = kw.xget('title','<NO TITLE>')
label = kw.xget('label','nolabel')
tkind = kw.xget('toc kind',kind)
xstr = ""
if not re.search('^page$',kind,re.I):
(datareqs,processors,outputs) = e4t.cli.get_elem_configs(*args,**kw)
for ou in outputs:
if not exists(ou):
ou = join(options.process_path,ou)
if not exists(ou):
logger.error('E01#001: Il file di specifica del processore %s non esiste',pr)
raise ValueError, 'E:MKEL:001'
wcfg = udict(cfg2hash(ou))
if wcfg:
lcfg=udict(wcfg.xget(label))
if lcfg:
if re.match('^multiple$',kind,re.I):
els = lcfg.xget_list('elements')
for el in els:
clcfg=udict(wcfg.xget(el))
num = clcfg.xget('num',num)
title = clcfg.xget('title',title)
tkind = clcfg.xget('kind',tkind)
label=el.lower()
xstr += "\\sriel{%s}{%s}{%s}{%s}\n" % (tkind.title(),num.upper(),title,label)
num = None
else:
num = lcfg.xget('num',num)
title = lcfg.xget('title',title)
tkind = lcfg.xget('kind',tkind)
if num:
xstr += "\\sriel{%s}{%s}{%s}{%s}" % (tkind.title(),num.upper(),title,label)
return xstr
def getDB(backend=None,config=None,session=None):
"""
Return DB Object for backend specified in the backend parameter or
in the config file (key 'backend') or in the session specified
@param backend: backend name (db2,mysql,oracle)
@type backend: string or None
@param config: configuration file (default ~/.db) or configuration dictionary
@type config: filename or dictionary or None
@param session: session into configuration dictionary
@type session: string or None
@raises BackendDBError: no backend
"""
if config is None:
config = cfg2hash(expanduser('~/.db'),
secure=True)
if session is not None:
if config.has_key(session):
config = config[session]
if backend is None:
if config.has_key('backend'):
backend = config['backend']
if backend is None:
raise BackendDBError('No Backend')
if re.match('^db2$',backend,re.I):
return DB2(config)
if re.match('^mysql$',backend,re.I):
return MYSQL(config)
if re.match('^oracle$',backend,re.I):
return ORACLE(config)
raise BackendDBError('Backend %s unknown' % backend)
def _read_specfile(spec):
if not exists(spec):
logger.error('Il file di specifica (%s) della table non esiste',spec)
return (-10,None,None)
base = dirname(spec) # jobs directory
logger.debug('carica il file di specifica da %s',spec)
#report.info(open(spec,'r').read())
jobs_specfile = cfg2hash(spec)
if hasattr(options,'model_check') and options.model_check:
logger.debug('Checking Model %s',spec)
names = ('BASE',)
if options.name:
names = options.name
for name in names:
logger.debug('checking %s',name)
errs = check(options,name,jobs_specfile,'jobspec.jobspec',filename=spec)
if errs:
print "ERROR SPECFILE/1"
sys.exit(print_msgs(options,errs))
print "ERROR SPECFILE"
sys.exit(0)
return (0,base,jobs_specfile)
def openDB(conf_filename=None,session_name="test",secure=True):
"""
Open database in the session from the configuration file
@param conf_filename: filename of configuration file
@param session_name: name of the session (defaults to 'test')
@param secure: configuration file is protected
@returns: DB object
@raises: NoConfigFileError
@raises: NoSessionInConfigFileError
"""
if conf_filename is None:
BASE_PATH=expanduser("~")
conf_filename = join(BASE_PATH,'.db')
logger.debug("Got ~/.db configuration file")
if not exists(conf_filename): conf_filename = None
if conf_filename is None:
BITS_PATH="/home/group/public/bits"
BITS_ETC=join(BITS_PATH,"etc")
conf_filename = join(BITS_ETC,'db.conf')
logger.debug("Got %s configuration file",conf_filename)
conf_file=expanduser(conf_filename)
if not exists(conf_file):
raise NoConfigFileError(conf_file)
config = cfg2hash(conf_file,secure=secure)
session = session_name
if config.has_key(session):
cfg = config[session]
db = getDB(config=cfg,session=session)
else:
raise NoSessionInConfigFileError(session,config_file)
return (db,cfg)
def check_conffile(icfg,options,conf,cfg,*args):
logger.debug("{CONFFILE} %s",args[0])
label = cfg.get('LABEL',cfg['__LABEL'])
base = dirname(cfg['__FILE'])
results=[]
for _n,(_f,_t) in args[0].items():
if cfg.has_key(_n.upper()):
fname= cfg[_n.upper()]
conffile = _i(fname,base)
if conffile:
if re.search("\.req$",conffile,re.I):
logger.debug('{CONFFILE} req %s,%s',fname,conffile)
pass
else:
logger.debug('{CONFFILE} conf %s,%s',fname,conffile)
ncfg = cfg2hash(conffile)
if _n=='processor':
label='PROCESSOR'
results.extend(check(options,label,ncfg,_t))
else:
kw = udict(cfg)
kw.update({'OFFENDING':fname})
results.append(('E:FILE:001',kw),)
return results
def writeout_joblist_file(elem,dname,pname,options=Options()):
"""Write a joblist file from a data-request and processor file
:param elem: output specification
:kind elem: string or filename
:param dname: data-request file name
:kind dname: filename
:param pname: processor step file name
:kind pname: filename
if elem is none then set up only dataset (and processor, if any) steps in joblist.
"""
def _get_elem_kind(conf):
if re.match('^T_',conf):
return 'table'
elif re.match('^F_',conf):
return 'figure'
return 'composite'
print "E=",elem
if not elem: # dataset
_D = { 'REQ' : {
'active' : 'yes',
'kind' : 'figure',
'panels': 'A11',
'datareq': dname
},
}
_D['A11'] = {
'SERIES': 'OUTS'
}
ofile = writeout_file(_D,header='# output file',options=options)
_d = { 'dataset': 'output.xls',
'kind': 'figure',
'output': ofile
}
if pname:
_d['processor']=pname
_dict={ 'BASE' : { 'jobs': 'JOBS' } , 'JOBS': { 'JOBS':'REQ', 'KIND': 'joblist' }, 'REQ' : _d }
setattr(options,'load',True)
return writeout_file(_dict,header='# joblist file',options=options)
kind = 'figure'
# Se esiste un output file
_dict = {}
for i,e in enumerate(elem):
print i,e
conf = None
if is_dict(e):
# l'output file deve essere generato dell'opzione stessa
_d = udict(as_dict(e))
if not nested_dict(_d):
# opzione semplice (es. series=A,B,C)
# Aggiungo
_D = { 'REQ' : {
'active' : 'yes',
'kind' : 'figure',
'panels': 'A11'
},
}
_D['A11'] = _d
_d = _D
label = 'REQ'
_dict.update(_d)
else:
if ':' in e:
if e.count(':')==2:
(kind,label,conf) = e.split(':')
elif e.count(':')==1:
(label,conf) = e.split(':')
kind = _get_elem_kind(conf)
elif e.count(':')>2:
raise ValueError, e
if conf:
_dict = cfg2hash(conf)
options.process_path=dirname(conf)
if conf is None:
conf = e
kind = _get_elem_kind(conf)
m = re.search('[TFC]_([a-zA-Z0-9_]+)$',conf)
if m:
label = m.group(1)
else:
label = "ELEM"
_d = { 'kind' : kind,
'label': label,
'output' : conf,
}
_dict[label]=_d
oname = writeout_file(_dict,header='# output file',options=options)
if 'OUT' in options.look:
lookout(oname,'Configurazione di output',options=options)
_d = udict({
'dataset': 'output.xls' ,
'kind': kind,
'output': '%s' % oname,
})
if dname:
_d['datareq']=dname
if pname:
_d['processor']=pname
_dict={ 'BASE' : { 'jobs': 'JOBS' } , 'JOBS': { 'JOBS':label, 'KIND': 'joblist', 'files': 'sristat.sty' }, label : _d }
return writeout_file(_dict,header='# joblist file',options=options)
def _output_element(options,name,specs,_dataset,elem_class,fname,**kw):
"""Crea il file tex per l'output del dataset a partire dal file conf"""
_accounting['output.name'] = name
_accounting['output.class'] = elem_class
_accounting['output.datalen'] = len(_dataset) if _dataset else 0
logger.debug('Esportazione documento %s con classe %s (Dlen=%s)',name,elem_class,_dataset)
if elem_class is None:
return
## Lettura del file di specifica
#
for spec in specs.split(','):
if re.search('\.req$',spec):
continue
if not exists(spec):
spec = join(options.process_path,spec)
if not exists(spec):
logger.error('E01#001: Il file di specifica dell\'elemento %s non esiste (%s)',name,spec)
raise ValueError, 'E:MKEL:001'
if exists(spec):
spec = cfg2hash(spec)
kw=udict(kw)
if 'macros' in kw:
spec['macros']=kw['macros']
else:
logger.error('{OUTPUTELEMENT}:E:INIT:001 -- Il file di configurazione %s non esiste', spec)
raise ValueError, "E:OUTPUTELEMENT:INIT:001"
try:
spec = mapspec(name,spec,_dataset,options,
lambda k,v,name,spec,_dataset,options:
(k, expandfuncs(v,name,spec,_dataset,options)))
except IndexError, exc:
logger.error('{OUTPUTELEMENT}:E:MAPSPEC:001 -- non posso espandere le funzioni')
raise
# test = expandfuncs('%TEST()',name,spec,_dataset,options)
with Timer() as T:
t = elem_class(name,spec,_dataset,options)
_accounting['output.time.setup']=T.msecs
if hasattr(t,'produce'):
with Timer() as T:
t.produce()
_accounting['output.time.produce']=T.msecs
l = LaTeX(options)
# if False and hasattr(options,'switch_components') and options.switch_components:
# try:
# output = join(options.output_path,fname.replace('.tex','.pdf'))
# logger.debug('Compilazione componente %s',fname)
# with Timer() as T:
# l.compile(name,t,fname=output)
# _accounting['output.time.latex.compile']=T.msecs
# _accounting['output.time.latex.output']=output
# outfiles.append(output)
# except IOError, exc:
# logger.error('Non posso scrivere il file %s',output)
_accounting['output.outfile']=fname
with Timer() as T:
ofile = fname
f = open(ofile, 'w')
f.write(l.layout(t,False))
f.close()
outfiles.append(ofile)
_accounting.update(l.report())
_accounting['output.time.write.layout']=T.msecs
if options.switch_debug:
# Write the report file
codecs.open("%s.rep"%name,'w','utf-8').write(t.report())
fname = "%s.out" % name
logger.debug('Write out %s',fname)
save_dict(fname,_accounting,options.switch_debug)
_accounting.clear()
def exec_show_job_SOLR(options, cwriter, *args, **kw):
global SOURCE, TITLE, SUBTITLE, PTITLE, LABEL, DATA, FORMULA, URL, PROCESSED, DOWNLOADED, DATAREQ, PROCESSOR, PID
if re.match("^(|us|jp|uk|in|ch|br|ru|po|tu)struct|cfa|cpt|uslms$", args[0], re.I):
return
setattr(options, "job", args[0])
kw = udict(kw)
base = join(options.process_path, "input")
(datareqs, processors, outputs) = e4t.cli.get_elem_configs(*args, **kw)
if len(datareqs) == 0:
return
acct = {}
DATAREQ = datareqs[0].replace(options.process_path, "") if datareqs[0] else ""
_dataset = tsload_datareqs(options, None, *datareqs, **kw)
prcfg = {}
if len(processors) > 0:
pr = processors[0]
PROCESSOR = pr
prcfg = udict()
if exists(pr):
pr = join(options.process_path, pr)
if exists(pr):
prcfg = cfg2hash(pr).xget_dict("PROCESSOR")
ou = outputs[0]
if not exists(ou):
ou = join(options.process_path, ou)
if not exists(ou):
logger.error("E01#001: Il file di specifica del processore %s non esiste", pr)
raise ValueError, "E:MKEL:001"
wcfg = cfg2hash(ou)
jcfg = wcfg.xget_dict(options.job)
SOURCE = clean_latex(jcfg.xget("SOURCE").replace("\n", " ") if "SOURCE" in jcfg else None)
TITLE = clean_latex(jcfg.xget("TITLE").replace("\n", " ") if "TITLE" in jcfg else None)
SUBTITLE = clean_latex(jcfg.xget("SUBTITLE").replace("\n", " ") if "SUBTITLE" in jcfg else None)
KIND = jcfg.xget("KIND")
if not re.match("figure|table", KIND, re.I):
return
_base_ds = set(_dataset.keys())
_dataset = process_dataset(options, _dataset, processors, *args, **kw)
_base_acct = udict()
nacct = rework_acct()
_base_acct.update(nacct)
_proc_ds = set(_dataset.keys()) - _base_ds
if re.match("figure", KIND, re.I):
panels = jcfg.xget_list("PANELS")
for panel in panels:
pcfg = wcfg.xget_dict(panel)
PTITLE = pcfg.xget("TITLE").replace("\n", " ") if "TITLE" in pcfg else None
PTITLE = clean_latex(PTITLE)
series = []
series1 = pcfg.xget_list("SERIES")
series2 = pcfg.xget_list("LEFT SERIES")
series3 = pcfg.xget_list("RIGHT SERIES")
if series1:
series.extend(series1)
if series2:
series.extend(series2)
if series3:
series.extend(series3)
for serie in series:
LABEL = serie
DATA = serie
FORMULA = None
URL = ""
PROVIDER = ""
scfg = wcfg.xget_dict(serie)
if scfg:
LABEL = clean_latex(scfg.xget("LABEL"))
DATA = scfg.xget("DATA")
PROCESSED = DATA in _proc_ds
DOWNLOADED = DATA in _base_ds
if PROCESSED:
FORMULA = prcfg.xget(DATA).replace("\n", " ") if DATA in prcfg else None
if DOWNLOADED:
if DATA not in _base_acct:
print DATA, "not in acct", _base_acct.keys(), ",".join(_base_ds)
else:
D = _base_acct.xget(DATA)
P = D["provider"]
URL = P + "://" + _base_acct.xget(DATA)["url"]
PROVIDER = P
PROCESSED = str(PROCESSED) if PROCESSED else None
DOWNLOADED = str(DOWNLOADED) if DOWNLOADED else None
WPROCESSOR = "" if not PROCESSOR else WSOURCE + PROCESSOR.replace(options.process_path, "")
XLSRAW = WEB + "ds/%s-raw.xls" % JOB.lower()
XLSPROC = WEB + "ds/%s-proc.xls" % JOB.lower() if PROCESSOR else ""
XTRNAL = "True" if re.match("^dstream|http|file", PROVIDER) is not None else None
PID += 1
cwriter.writerow(
(
str(PID),
PUBLIC,
str(PAGE),
KIND,
JOB,
WEB + OUTFILE,
WSOURCE + DATAREQ,
WPROCESSOR,
WSOURCE + OUTPUT,
XLSRAW,
XLSPROC,
TITLE,
SUBTITLE,
SOURCE,
PTITLE,
LABEL,
DATA,
DOWNLOADED,
PROVIDER,
URL,
PROCESSED,
FORMULA,
XTRNAL,
)
)
pass
elif re.match("table", KIND, re.I):
blocks = jcfg.xget_list("BLOCK")
if not blocks:
return
orient, m, n, ELS = _extract_data_matrix_def(prcfg, _dataset)
line = 0
TITLES = []
for b in blocks:
bcfg = wcfg.xget_dict(b)
models = bcfg.xget_list("MODEL")
PTITLE = bcfg.xget("TITLE").replace("\n", " ") if "TITLE" in bcfg else None
if PTITLE:
PTITLE = clean_latex(PTITLE)
else:
PTITLE = TITLE
for mod in models:
mod = clean_model(mod)
if not mod:
continue
LABEL = mod
DATA = mod
FORMULA = None
URL = ""
PROVIDER = ""
scfg = wcfg.xget_dict(mod)
if scfg:
TITLES.append(clean_latex(scfg.xget("TITLE")))
line += 1
lines = line
for line in range(lines):
series = ELS[line]
for n, serie in series:
DATA = serie
PROCESSED = DATA in _proc_ds
DOWNLOADED = DATA in _base_ds
if PROCESSED:
FORMULA = prcfg.xget(DATA).replace("\n", " ") if DATA in prcfg else None
if DOWNLOADED:
if DATA not in _base_acct:
print DATA, "not in acct", _base_acct.keys(), ",".join(_base_ds)
else:
D = _base_acct.xget(DATA)
P = D["provider"]
URL = P + "://" + _base_acct.xget(DATA)["url"]
PROVIDER = P
PROCESSED = str(PROCESSED) if PROCESSED else None
DOWNLOADED = str(DOWNLOADED) if DOWNLOADED else None
WPROCESSOR = "" if not PROCESSOR else WSOURCE + PROCESSOR.replace(options.process_path, "")
XLSRAW = WEB + "ds/%s-raw.xls" % JOB.lower()
XLSPROC = WEB + "ds/%s-proc.xls" % JOB.lower() if PROCESSOR else ""
XTRNAL = "True" if re.match("^dstream|http|file", PROVIDER) is not None else None
TITLE = TITLES[line]
PID += 1
cwriter.writerow(
(
str(PID),
PUBLIC,
str(PAGE),
KIND,
JOB,
WEB + OUTFILE,
WSOURCE + DATAREQ,
WPROCESSOR,
WSOURCE + OUTPUT,
XLSRAW,
XLSPROC,
TITLE,
SUBTITLE,
SOURCE,
PTITLE,
LABEL,
DATA,
DOWNLOADED,
PROVIDER,
URL,
PROCESSED,
FORMULA,
XTRNAL,
)
)
elif re.match("multiple", KIND, re.I):
pass
else:
logger.error("E01#002: Il file di specifica del processore %s non esiste", pr)
raise ValueError, "E:MKEL:001"
def __init__(self,options,cfg,gspec=None,spec=None):
if gspec is None:
gspec = spec = cfg2hash(cfg)
ConfigElem.__init__(self,options,cfg,gspec,spec)
self._base = {}
def exec_job_WIKI(options,*args,**kw):
kw = udict(kw)
kind = kw.xget('KIND')
if kind is None:
return
num = kw.xget('num')
title = kw.xget('title','<NO TITLE>')
label = kw.xget('label','nolabel')
tkind = kw.xget('toc kind',kind)
home = dirname(options.jobfile)
tfile = kw.xget('__file')
if tfile:
tfile=tfile.replace(home,'')
tfile=dirname(tfile)
else:
tfile=""
xstr = ""
if not re.search('^page|cover|empty$',kind,re.I):
(datareqs,processors,outputs) = e4t.cli.get_elem_configs(*args,**kw)
for ou in outputs:
if not exists(ou):
ou = join(options.process_path,ou)
if not exists(ou):
logger.error('E01#001: proc spec file %s not exists',pr)
raise ValueError, 'E:MKEL:001'
wcfg = udict(cfg2hash(ou))
if wcfg:
lcfg=udict(wcfg.xget(label,{}))
if lcfg:
if re.match('^multiple$',kind,re.I):
els = lcfg.xget_list('elements')
for _i,el in enumerate(els):
clcfg=udict(wcfg.xget(el))
num = lcfg.xget('num',num)
title = clcfg.xget('title',title)
tkind = clcfg.xget('kind',tkind)
tlabel=el.lower()
tfile = clcfg.xget('__file')
if tfile:
tfile=tfile.replace(home,'')
tfile=dirname(tfile)
else:
tfile=""
print tfile
xstr += '||%s||%s||%s||%s||%s||%s||%s||'%(label,num,
_wiki_pdflink(options.web,options.work,options.phase,options.version,label,title),
_svn_conflink(tfile,'SRC'),
_wiki_xlslink(options,kw,options.output_path,options.web,options.work,options.phase,options.version,label,title),
_trac_tickets('this',label),
_trac_tickets('next',label) )
num = None
if _i<=(len(els)-1): xstr += '\n'
else:
num = lcfg.xget('num',num)
title = lcfg.xget('title',title)
tkind = lcfg.xget('kind',tkind)
tfile = lcfg.xget('__file')
if tfile:
tfile=tfile.replace(home,'')
tfile=dirname(tfile)
else:
tfile=""
title = re.sub('.tmark\[[^\]]\]','',title,re.I)
if num:
if num!="0":
xstr = '||%s||%s||%s||%s||%s ||%s||%s||'%(label,num,
_wiki_pdflink(options.web,options.work,options.phase,options.version,label,title),
_svn_conflink(tfile,'SRC'),
_wiki_xlslink(options,kw,options.output_path,options.web,options.work,options.phase,options.version,label,title),
_trac_tickets('this',label),
_trac_tickets('next',label),
)
return xstr