def shelved2onlinedb(self):
if len(self.shelveddb) > 0:
csvfilepath = None
curs = None
try:
fd, csvfilepath = tempfile.mkstemp(suffix='.csv',
prefix=self.name)
tmpfile = file(csvfilepath, 'w')
csvwriter = UnicodeWriter(tmpfile, delimiter='\t')
for k in self.shelveddb:
csvwriter.writerows(self.shelveddb[k])
tmpfile.close()
os.close(fd)
curs = self.con.cursor()
curs.execute('TRUNCATE %s' % self.name)
curs.copy_from(file=file(csvfilepath),
table=self.name,
sep='\t',
null='',
columns=self.all)
finally:
if curs:
curs.close()
os.remove(csvfilepath)
a.extend(row.xpath(xx))
if a:
# print "matched"
rd[k]="".join([stringstring(ss) for ss in a])
if len(rd[k]) == 1 :
rd[k]=rd[k][0]
of=False
props.update(rd)
for o in parse["ok"] :
if output_ok(rd,o) :
cdata.append(output_row(props,parse["table"]))
of=True
break
# if not of:
# rd.update({"file" : "o"})
# cdata.append(output_row(rd,parse["table"]))
drows=drows+len(cdata)
data.append(cdata)
except Exception,e:
print "%s - error %s" % (fn,e)
else :
print "%s - success - %s tables, %s records" % (fn,len(data),drows)
# pprint.pprint(map(lambda a: { "a" : a.get("reaction","-"), "b" : a.get("color",""), "c" : a.get("amendment","") },data));
for container in data :
if len(container)>0 :
ofile.writerows(container)
def output_csv(rows):
global args
writer = UnicodeWriter(sys.stdout)
writer.writerows(rows)
res.append(r)
return res
rr=[]
for f in input_files.split("\n")[:5] :
try :
n=parsefile(f)
if len(n)>0 :
rr.extend(n)
print "%s - %s" % (len(n),f)
else :
print "!! - %s" % (f,)
except Exception, e:
raise
print "!!! - %s - %s" % (f,e)
import simplejson,string
rr_germany=filter(lambda a : string.lower("%(category)s %(share)s" % a).find("germany")>-1,rr)
simplejson.dump(rr,open(os.path.join(output_dir,"resultate-runde2.json"),"w"))
simplejson.dump(rr_germany,open(os.path.join(output_dir,"resultate-runde2-germany.json"),"w"))
from unicodecsv import UnicodeWriter
uw=UnicodeWriter(open(os.path.join(output_dir,"resultate-runde2.csv"),"w"),delimiter=";")
uw.writerows(map(lambda a: [a["date"],a["category"],a.get("fund",""),a["share"],a["number"],a["price"],a["file"]], rr))
uw=UnicodeWriter(open(os.path.join(output_dir,"resultate-runde2-germany.csv"),"w"),delimiter=";")
uw.writerows(map(lambda a: [a["date"],a["category"],a.get("fund",""),a["share"],a["number"],a["price"],a["file"]], rr_germany))
