def shelved2onlinedb(self): if len(self.shelveddb) > 0: csvfilepath = None curs = None try: fd, csvfilepath = tempfile.mkstemp(suffix='.csv', prefix=self.name) tmpfile = file(csvfilepath, 'w') csvwriter = UnicodeWriter(tmpfile, delimiter='\t') for k in self.shelveddb: csvwriter.writerows(self.shelveddb[k]) tmpfile.close() os.close(fd) curs = self.con.cursor() curs.execute('TRUNCATE %s' % self.name) curs.copy_from(file=file(csvfilepath), table=self.name, sep='\t', null='', columns=self.all) finally: if curs: curs.close() os.remove(csvfilepath)
a.extend(row.xpath(xx)) if a: # print "matched" rd[k]="".join([stringstring(ss) for ss in a]) if len(rd[k]) == 1 : rd[k]=rd[k][0] of=False props.update(rd) for o in parse["ok"] : if output_ok(rd,o) : cdata.append(output_row(props,parse["table"])) of=True break # if not of: # rd.update({"file" : "o"}) # cdata.append(output_row(rd,parse["table"])) drows=drows+len(cdata) data.append(cdata) except Exception,e: print "%s - error %s" % (fn,e) else : print "%s - success - %s tables, %s records" % (fn,len(data),drows) # pprint.pprint(map(lambda a: { "a" : a.get("reaction","-"), "b" : a.get("color",""), "c" : a.get("amendment","") },data)); for container in data : if len(container)>0 : ofile.writerows(container)
def output_csv(rows): global args writer = UnicodeWriter(sys.stdout) writer.writerows(rows)
res.append(r) return res rr=[] for f in input_files.split("\n")[:5] : try : n=parsefile(f) if len(n)>0 : rr.extend(n) print "%s - %s" % (len(n),f) else : print "!! - %s" % (f,) except Exception, e: raise print "!!! - %s - %s" % (f,e) import simplejson,string rr_germany=filter(lambda a : string.lower("%(category)s %(share)s" % a).find("germany")>-1,rr) simplejson.dump(rr,open(os.path.join(output_dir,"resultate-runde2.json"),"w")) simplejson.dump(rr_germany,open(os.path.join(output_dir,"resultate-runde2-germany.json"),"w")) from unicodecsv import UnicodeWriter uw=UnicodeWriter(open(os.path.join(output_dir,"resultate-runde2.csv"),"w"),delimiter=";") uw.writerows(map(lambda a: [a["date"],a["category"],a.get("fund",""),a["share"],a["number"],a["price"],a["file"]], rr)) uw=UnicodeWriter(open(os.path.join(output_dir,"resultate-runde2-germany.csv"),"w"),delimiter=";") uw.writerows(map(lambda a: [a["date"],a["category"],a.get("fund",""),a["share"],a["number"],a["price"],a["file"]], rr_germany))