def main(): global log cfg_file = "tevs.cfg" out_file = open("summary.csv","w") config.get(cfg_file) log = config.logger(util.root("log.txt")) #pdb.set_trace() #matches = is_this_like_them("hello there",{"hellc there":"bonjour","yabba":"dabba"}) if not const.use_db: print "The configuration file indicates no database is in use." print "We will now build totals from the results files." build_totals_from_results_files() output_totals_from_results_files() return 0 try: dbc = db.PostgresDB(const.dbname, const.dbuser) print "Generating totals from db %s, user %s" % (const.dbname, const.dbuser) print "If there are many vote ops, this may take several minutes." print "The next output will be the number of vote opportunities." qs = query_database(dbc) except db.DatabaseError: print "Although the configuration file indicates a database is in use," print "we could not connect for dbname %s user %s." % (const.dbname, const.dbuser) print "We will now build totals from the results files." build_totals_from_results_files() output_totals_from_results_files() return 0 return 0
def main(): global log cfg_file = "tevs.cfg" out_file = open("summary.csv", "w") config.get(cfg_file) log = config.logger(util.root("log.txt")) #pdb.set_trace() #matches = is_this_like_them("hello there",{"hellc there":"bonjour","yabba":"dabba"}) if not const.use_db: print "The configuration file indicates no database is in use." print "We will now build totals from the results files." build_totals_from_results_files() output_totals_from_results_files() return 0 try: dbc = db.PostgresDB(const.dbname, const.dbuser) print "Generating totals from db %s, user %s" % (const.dbname, const.dbuser) print "If there are many vote ops, this may take several minutes." print "The next output will be the number of vote opportunities." qs = query_database(dbc) except db.DatabaseError: print "Although the configuration file indicates a database is in use," print "we could not connect for dbname %s user %s." % (const.dbname, const.dbuser) print "We will now build totals from the results files." build_totals_from_results_files() output_totals_from_results_files() return 0 return 0
def main(counter, duplex, comment, inches, resolution): # read configuration from tevs.cfg and set constants for this run const.debug = False #XXX config.get("tevs.cfg") util.mkdirp(const.root) log = config.logger(util.root("scan.log")) inches_to_mm = 25.4 inc = 1 if duplex: inc = 2 num = next.IncrementingFile(util.root("nexttoscan.txt"), inc) try: scanner = Scanner( duplex, int(inches * inches_to_mm), resolution ) while True: counter = num.value() print "Scanning",counter stamp = datetime.now().isoformat() for i, img in enumerate(scanner.scan(counter)): #get path n = counter + i p = "%03d" % (n/1000,) f = "%06d" % n dir = util.root(const.incoming, p) util.mkdirp(dir) filename = os.path.join(dir, f + ".jpg") img.save(filename) print "Saved",filename log.info("Saved %s at %s\n%s", filename, stamp, comment) num.increment_and_save() except ScanningException: print "Empty feeder?" log.info("Scan aborted due to empty feeder for 20 seconds.") sys.exit(2) except KeyboardInterrupt: log.info("Scan aborted by user") sys.exit(1)
def mark_error(e, *files): log = logging.getLogger('') if e is not None: log.error(e) for file in files: log.error("Could not process ballot " + os.path.basename(file)) try: shutil.copy2(file, util.root("errors", os.path.basename(file))) except IOError: log.error("Could not copy unprocessable file to errors dir") return len(files)
def main(counter, duplex, comment, inches, resolution): # read configuration from tevs.cfg and set constants for this run const.debug = False #XXX config.get("tevs.cfg") util.mkdirp(const.root) log = config.logger(util.root("scan.log")) inches_to_mm = 25.4 inc = 1 if duplex: inc = 2 num = next.IncrementingFile(util.root("nexttoscan.txt"), inc) try: scanner = Scanner(duplex, int(inches * inches_to_mm), resolution) while True: counter = num.value() print "Scanning", counter stamp = datetime.now().isoformat() for i, img in enumerate(scanner.scan(counter)): #get path n = counter + i p = "%03d" % (n / 1000, ) f = "%06d" % n dir = util.root(const.incoming, p) util.mkdirp(dir) filename = os.path.join(dir, f + ".jpg") img.save(filename) print "Saved", filename log.info("Saved %s at %s\n%s", filename, stamp, comment) num.increment_and_save() except ScanningException: print "Empty feeder?" log.info("Scan aborted due to empty feeder for 20 seconds.") sys.exit(2) except KeyboardInterrupt: log.info("Scan aborted by user") sys.exit(1)
def main(counter, duplex, comment, inches, resolution): # read configuration from tevs.cfg and set constants for this run const.debug = False #XXX config.get("tevs.cfg") util.mkdirp(const.root) log = config.logger(util.root("scan.log")) inches_to_mm = 25.4 print counter inc = 1 if duplex: inc = 2 num = next.Simple(counter, inc) if counter < 0: num = next.File(util.root("nexttoscan.txt"), inc) try: scanner = Scanner(duplex, int(inches * inches_to_mm), resolution) for counter in num: stamp = datetime.now().isoformat() for i, img in enumerate(scanner.scan(counter)): #get path n = counter + i p = "%03d" % (n / 1000, ) f = "%06d" % n dir = util.root(const.incoming, p) util.mkdirp(dir) file = os.path.join(dir, f + ".jpg") img.save(file) log.info("Saving %s at %s\n%s", file, stamp, comment) num.save() except KeyboardInterrupt: log.info("Scan aborted by user") sys.exit(1)
def executeQuery(sparqlEndpoint, query, returnFormat = JSON): """ Execute SPARQL @query on @sparqlEndpoint. Cache the query results temporarily in the tmp directory. """ path = os.path.join(util.root(), "tmp", util.sha1(sparqlEndpoint.endpoint + query)) if os.path.exists(path): with open(path, "rb") as file: return pickle.load(file) else: sparqlEndpoint.setQuery(unicode(query.decode("UTF-8"))) sparqlEndpoint.setReturnFormat(returnFormat) results = sparqlEndpoint.query().convert() with open(path, "wb") as file: pickle.dump(results, file) return results
def main(): cfg_file = "tevs.cfg" out_file = open("summary.csv", "w") config.get(cfg_file) log = config.logger(util.root("log.txt")) if not const.use_db: log.error("Cannot summarize database without a database") return 1 try: dbc = db.PostgresDB(const.dbname, const.dbuser) print "preparing summary for", const.dbname, "user", const.dbuser except db.DatabaseError: log.error("Could not connect to database") return 2 qs = query(dbc, out_file) pdb.set_trace() printer = display['console'] print printer(qs) return 0
def __init__(self, x1, y1, x2, y2, v_margin=1, h_margin=1, side="Noside", layout_id="Nolayoutid", image=None, image_filename="Nofilename", jurisdiction="Nojurisdiction", contest="Nocontest", choice="Nochoice", max_votes=1, logger=None): self.x1 = x1 self.y1 = y1 self.x2 = x2 self.y2 = y2 logger.info("VOP at %d %d %d %d" % (self.x1, self.y1, self.x2, self.y2)) self.crop_bbox = (x1 - h_margin, y1 - v_margin, x2 + h_margin, y2 + v_margin) # test crop_bbox for reasonableness, and recenter if necessary # report recentered coordinates as adjusted_x and y. self.image = image self.image_filename = image_filename self.side = side self.layout_id = layout_id self.jurisdiction = jurisdiction self.contest = contest self.choice = choice self.histogram = None self.red_mean = None self.red_lowest = None self.red_low = None self.red_high = None self.red_highest = None self.green_mean = None self.green_lowest = None self.green_low = None self.green_high = None self.green_highest = None self.blue_mean = None self.blue_lowest = None self.blue_low = None self.blue_high = None self.blue_highest = None self.voted = None self.ambiguous = None self.max_votes = max_votes self.logger = logger # overvoted flag is set by BallotSideWalker # when it has finished checking all votes for a box self.overvoted = False # vote test sets self.voted and self.ambiguous # returns 0 on success but 1 if updating self coordinates self.crop = image.crop(self.crop_bbox) self.stat = ImageStat.Stat(self.crop) # fills in r/g/b mean values self.mean() # make a copy of self.mean, and throw out the adjustment # if this copy is darker than the new copy (orig_red_mean, orig_green_mean, orig_blue_mean) = self.stat.mean # !!!new mjt 11/12/13 ret_coord = coord_adjust.coord_adjust(self.crop, h_margin, v_margin, x2 - x1, y2 - y1) if ret_coord is not None: #print ret_coord, "applies to old crop now saved." #self.crop.save("/tmp/saved.jpg") # the returned coordinate is the ULC in a crop box w/ margins self.x1 = x1 + ret_coord[0] - h_margin self.y1 = y1 + ret_coord[1] - v_margin self.x2 = x2 + ret_coord[0] - h_margin self.y2 = y2 + ret_coord[1] - v_margin logger.info("VOP adj to %d %d %d %d" % (self.x1, self.y1, self.x2, self.y2)) #print "VOP adj to %d %d %d %d" % (self.x1,self.y1,self.x2,self.y2) self.crop_bbox = (self.x1 - h_margin, self.y1 - v_margin, self.x2 + h_margin, self.y2 + v_margin) self.crop = image.crop(self.crop_bbox) #self.crop.save("/tmp/saved_adj.jpg") #pdb.set_trace() self.stat = ImageStat.Stat(self.crop) # fills in r/g/b mean values self.mean() # fills in r/g/b lowest/low/high/highest values # undo move if it turns out this red_mean is lighter than orig_red_mean if orig_red_mean < self.red_mean: self.x1 = x1 self.y1 = y1 self.x2 = x2 self.y2 = y2 self.crop_bbox = (x1 - h_margin, y1 - v_margin, x2 + h_margin, y2 + v_margin) self.crop = image.crop(self.crop_bbox) self.stat = ImageStat.Stat(self.crop) # fills in r/g/b mean values self.mean() logger.info("VOP unadj back to %d %d %d %d" % (self.x1, self.y1, self.x2, self.y2)) self.hist4() self.vote_test() # save image around write-in if (self.choice.find("Write") >= 0) and (self.voted or self.ambiguous): #print "self.choice",self.choice #print "self.voted",self.voted #print "self.ambiguous",self.ambiguous # we should save the image with the writein to our writeins folder # using the contest text's first few chars, # first, generate a subimage using the upper left # of self.crop_bbox and extending for the contest width and # for 1". # !!!TODO # image_to_save = self.image.crop((self.crop_bbox[0], # self.crop_bbox[1], # self.crop_bbox[2]+(the column's width), # self.crop_bbox[3]+dpi)) outcropbox = None try: wizhop = const.writein_zone_horiz_offset_inches * const.dpi #self.logger.info("Wizhop %s" % (wizhop,)) wizvop = const.writein_zone_vert_offset_inches * const.dpi #self.logger.info("Wizvop %s" % (wizvop,)) wizwidth = const.writein_zone_width_inches * const.dpi #self.logger.info("Wizwidth %s" % (wizwidth,)) wizheight = const.writein_zone_height_inches * const.dpi #self.logger.info("Wizheight %s" % (wizheight,)) outfile = os.path.join( util.root("writeins"), self.contest[:4] + os.path.basename(image_filename)) outcropbox = (int(self.crop_bbox[0] + wizhop), int(self.crop_bbox[1] + wizvop), int(self.crop_bbox[0] + wizhop + wizwidth), int(self.crop_bbox[1] + wizvop + wizheight)) self.image.crop(outcropbox).save(outfile) self.logger.info("Wrote writein image %s" % (outfile, )) except Exception as e: self.logger.warning("Could not save writein image %s %s %s" % (outfile, outcropbox, e))
def merge_with_composites(self): """update composite images for this layout with this ballot's images""" try: oldimage = Image.open("%s/%s%d/%s.jpg" % ( util.root(), "composite_images",os.getpid(), tmpl.barcode)) except: try: oldimage = Image.open("%s/%s%d/%s.jpg" % ( util.root(), "template_images",os.getpid(), tmpl.barcode)) except: oldimage = None # else retrieve template image # derotate the new image as if you were building a template r2d = 180/3.14 newimage = page.image.rotate(-r2d * page.rot, Image.BILINEAR) #page.image.save("/tmp/postrotate.jpg") if oldimage is None: oldimage = newimage # landmarks will change once image is derotated! try: self.FindLandmarks(pagenum) except BallotException: pass # translate the new image to align with composite image delta_x = tmpl.xoff - page.xoff delta_y = tmpl.yoff - page.yoff newimage = newimage.offset(delta_x,delta_y) #newimage.save("/tmp/posttranslate.jpg") # apply darker operation, save result in first argument? oldimage.load() oldr,oldg,oldb = oldimage.split() newr,newg,newb = newimage.split() # count dark pixels in oldr excluding edges oldr_crop = oldr.crop((const.dpi/4, const.dpi/4, min(oldr.size[0],newr.size[0]) - (const.dpi/4), min(oldr.size[1],newr.size[1]) - (const.dpi/4))) old_total_intensity = 0 for p in oldr_crop.getdata(): old_total_intensity += p newr = ImageChops.darker(oldr,newr) newg = ImageChops.darker(oldg,newg) newb = ImageChops.darker(oldb,newb) new_total_intensity = 0 # count dark pixels in newr excluding edges newr_crop = newr.crop((const.dpi/4, const.dpi/4, min(newr.size[0],oldr.size[0]) - (const.dpi/4), min(newr.size[1],oldr.size[1]) - (const.dpi/4))) new_total_intensity = 0 for p in newr_crop.getdata(): new_total_intensity += p self.logger.info("%s Old %d New %d Diff %d" % ( os.path.basename(page.filename), old_total_intensity, new_total_intensity, old_total_intensity - new_total_intensity)) newimage = Image.merge("RGB",(newr,newg,newb)) try: composite_counts_dict[tmpl.barcode] += 1 if 0==(composite_counts_dict[tmpl.barcode]%5): self.logger.info( "Composite count for %s now %d (this run only)" % ( tmpl.barcode, composite_counts_dict[tmpl.barcode] )) except: composite_counts_dict[tmpl.barcode] = 1 newimage.save("%s/%s%d/%s.jpg" % ( util.root(), "composite_images",os.getpid(), tmpl.barcode))
def dirn(dir, n): # where dir is the "unrooted" name return util.root(dir, _fn(n))
def main(): miss_counter = 0 # get command line arguments cfg_file = get_args() # read configuration from tevs.cfg and set constants for this run config.get(cfg_file) util.mkdirp(const.root) log = config.logger(const.logfilename) #create initial top level dirs, if they do not exist for p in ("%s" % ("templates"), "%s%d" % ("template_images", os.getpid()), "%s%d" % ("composite_images", os.getpid()), "results", "proc", "errors"): util.mkdirp(util.root(p)) next_ballot = next.File(util.root("nexttoprocess.txt"), const.num_pages) try: ballotfrom = Ballot.LoadBallotType(const.layout_brand) except KeyError as e: util.fatal("No such ballot type: " + const.layout_brand + ": check " + cfg_file) # allow all instances to share a common template location, # though need per-pid locs for template_images and composite_images cache = Ballot.TemplateCache(util.root("templates")) extensions = Ballot.Extensions(template_cache=cache) # connect to db and open cursor if const.use_db: try: dbc = db.PostgresDB(const.dbname, const.dbuser) except db.DatabaseError: util.fatal("Could not connect to database") else: dbc = db.NullDB() total_proc, total_unproc = 0, 0 base = os.path.basename # While ballot images exist in the directory specified in tevs.cfg, # create ballot from images, get landmarks, get layout code, get votes. # Write votes to database and results directory. Repeat. #from guppy import hpy;hp=hpy();hp.setref();import gc;gc.disable();gc.collect();hp.setref() try: for n in next_ballot: gc.collect() unprocs = [incomingn(n + m) for m in range(const.num_pages)] if not os.path.exists(unprocs[0]): miss_counter += 1 log.info( base(unprocs[0]) + " does not exist. No more records to process") if miss_counter > 10: break continue #for i, f in enumerate(unprocs[1:]): # if not os.path.exists(f): # log.info(base(f) + " does not exist. Cannot proceed.") # for j in range(i): # log.info(base(unprocs[j]) + " will NOT be processed") # total_unproc += mark_error(None, *unprocs[:i-1]) #Processing log.info("Processing %s:\n %s" % (n, "\n".join("\t%s" % base(u) for u in unprocs))) try: ballot = ballotfrom(unprocs, extensions) results = ballot.ProcessPages() except BallotException as e: total_unproc += mark_error(e, *unprocs) log.exception("Could not process ballot") continue csv = Ballot.results_to_CSV(results) #moz = Ballot.results_to_mosaic(results) #Write all data #make dirs: proc1d = dirn("proc", n) resultsd = dirn("results", n) resultsfilename = filen(resultsd, n) for p in (proc1d, resultsd): util.mkdirp(p) try: results_to_vop_files(results, resultsfilename) except Exception as e: print e #write csv and mosaic util.genwriteto(resultsfilename + ".txt", csv) #write to the database try: dbc.insert(ballot) except db.DatabaseError: #dbc does not commit if there is an error, just need to remove #partial files remove_partial(resultsfilename + ".txt") remove_partial(resultsfilename + const.filename_extension) util.fatal("Could not commit vote information to database") #Post-processing # move the images from unproc to proc procs = [ filen(proc1d, n + m) + const.filename_extension for m in range(const.num_pages) ] for a, b in zip(unprocs, procs): try: os.rename(a, b) except OSError as e: util.fatal("Could not rename %s", a) total_proc += const.num_pages log.info("%d images processed", const.num_pages) #hp.heap().dump('prof.hpy');hp.setref();gc.collect();hp.setref();hp.heap().dump('prof.hpy') finally: cache.save_all() dbc.close() next_ballot.save() log.info("%d images processed", total_proc) if total_unproc > 0: log.warning("%d images NOT processed.", total_unproc)
def main(): NextEqualsPrefix = "Next=" MorePrompt = ":" NextToProcessFile = "" miss_counter = 0 # get command line arguments cfg_file = get_args() # read configuration from tevs.cfg and set constants for this run config.get(cfg_file) util.mkdirp(const.root) log = config.logger(util.root("extraction.log")) # create initial toplevel directories if they don't exist for p in ("%s" % ("templates"), "%s" % ("template_images"), "%s" % ("composite_images"), "results", "proc", "errors"): util.mkdirp(util.root(p)) # make sure you have code for ballot type spec'd in config file try: ballotfrom = Ballot.LoadBallotType(const.layout_brand) except KeyError as e: util.fatal("No such ballot type: %s check %s !", (const.layout_brand, cfg_file)) cache = Ballot.TemplateCache(util.root("templates")) extensions = Ballot.Extensions(template_cache=cache) # connect to db and open cursor if const.use_db: try: dbc = db.PostgresDB(database=const.dbname, user=const.dbuser) except db.DatabaseError: util.fatal("Could not connect to database!") else: dbc = db.NullDB() log.info("Database connected.") total_images_processed, total_images_left_unprocessed = 0, 0 base = os.path.basename # Each time given a signal to proceed for count_to_process ballots, # create ballot from images, get landmarks, get layout code, get votes. # Write votes to database and results directory. # for profiling # from guppy import hpy;hp=hpy();hp.setref(); # import gc;gc.disable();gc.collect();hp.setref() NextToProcessFile = util.root("nexttoprocess.txt") count_to_process = 0 file_problem = False while True: log.debug("Top of loop.") next_ballot_number = int(util.readfrom(NextToProcessFile)) log.debug("Read %d from %s" % (next_ballot_number, NextToProcessFile)) if count_to_process == 0: # send prompt to controlling process, "READY:" or "+ for SKIP:" if file_problem: file_problem = False # do not remove space after %06d print "Next=%06d , + to SKIP:" % (next_ballot_number, ) else: # do not remove space after %06d print "Next=%06d , READY:" % (next_ballot_number, ) sys.stdout.flush() # wait here until get_count_to_process returns # it will wait on input instruction from stdio try: count_to_process = get_count_to_process( next_ballot_number, log) except DoIncrementException, e: log.debug("Do increment exception") util.writeto(NextToProcessFile, next_ballot_number + const.num_pages) log.debug( "Wrote %d to next_ballot_number, count to process is %d" % (next_ballot_number + const.num_pages, count_to_process)) count_to_process = 0 log.debug("Setting count to process to 0.") continue # we're done when we get instructed to process 0 if count_to_process == 0: break count_to_process -= 1 try: # get number of next image, # clean up, in case... gc.collect() log.debug("Request for %d" % (next_ballot_number, )) unprocs = [ incomingn(next_ballot_number + m) for m in range(const.num_pages) ] log.info(unprocs) # we need all images for sheet to be available to process it for filename in unprocs: log.info("Checking for path.") if not os.path.exists(filename): log.info("File not present.") errmsg = "File %s not present or available!!!" % ( base(filename), ) log.info(errmsg.replace("!!!", "")) print errmsg sys.stdout.flush() raise FileNotPresentException(filename) log.info("Path found.") #Processing log.debug("Creating ballot.") try: ballot = ballotfrom(unprocs, extensions) log.debug("Created ballot, processing.") results = ballot.ProcessPages() log.debug("Processed.") except BallotException as e: total_images_left_unprocessed += mark_error(e, *unprocs) log.exception("Could not process ballot") util.writeto(NextToProcessFile, next_ballot_number + const.num_pages) continue #Write all data #make dirs: proc1d = dirn("proc", next_ballot_number) resultsd = dirn("results", next_ballot_number) resultsfilename = filen(resultsd, next_ballot_number) for p in (proc1d, resultsd): util.mkdirp(p) #try: # results_to_vop_files(results,resultsfilename) #except Exception as e: # log.info(e) # print e #write csv and mosaic #log.info("local results_to_CSV") #csv = results_to_CSV(results,log) #log.info("Back from results_to_CSV") #util.genwriteto(resultsfilename + ".csv", csv) #write to the database try: log.debug("Inserting to db") dbc.insert(ballot) except db.DatabaseError: #dbc does not commit if there is an error, just need to remove #partial files remove_partial(resultsfilename + ".txt") remove_partial(resultsfilename + const.filename_extension) log.info("Could not commit to db") print "Could not commit to db!" util.fatal("Could not commit vote information to database") #Post-processing # move the images from unproc to proc log.debug("Renaming") procs = [ filen(proc1d, next_ballot_number + m) + const.filename_extension for m in range(const.num_pages) ] for a, b in zip(unprocs, procs): try: os.rename(a, b) except OSError as e: log.info("Could not rename %s" % a) util.fatal("Could not rename %s", a) total_images_processed += const.num_pages # Tell caller you've processed all images of this ballot log.debug("Requesting next") util.writeto(NextToProcessFile, next_ballot_number + const.num_pages) # update next ballot file with next image number log.debug("Done writing nexttoprocess.txt") #print "%d extracted. " % (next_ballot_number,) log.info("%d images processed", const.num_pages) # for profiling # hp.heap().dump('prof.hpy');hp.setref();gc.collect(); # hp.setref();hp.heap().dump('prof.hpy') except FileNotPresentException, e: file_problem = True print "FileNotPresentException" sys.stdout.flush() log.info("FileNotPresentException occurred") continue
def main(): miss_counter = 0 # get command line arguments cfg_file = get_args() # read configuration from tevs.cfg and set constants for this run config.get(cfg_file) util.mkdirp(const.root) log = config.logger(const.logfilename) log.info("Log created.") # create initial toplevel directories if they don't exist for p in ("%s" % ("templates"), "%s" % ("template_images"), "%s" % ("composite_images"), "results", "proc", "errors"): util.mkdirp(util.root(p)) # make sure you have code for ballot type spec'd in config file try: ballotfrom = Ballot.LoadBallotType(const.layout_brand) except KeyError as e: util.fatal("No such ballot type: " + const.layout_brand + ": check " + cfg_file) cache = Ballot.TemplateCache(util.root("templates")) extensions = Ballot.Extensions(template_cache=cache) # connect to db and open cursor if const.use_db: try: dbc = db.PostgresDB(database=const.dbname, user=const.dbuser) except db.DatabaseError: util.fatal("Could not connect to database") else: dbc = db.NullDB() log.info("Database connected.") total_images_processed, total_images_left_unprocessed = 0, 0 base = os.path.basename # Each time given a signal to proceed for count_to_process ballots, # create ballot from images, get landmarks, get layout code, get votes. # Write votes to database and results directory. # for profiling # from guppy import hpy;hp=hpy();hp.setref(); # import gc;gc.disable();gc.collect();hp.setref() count_to_process = 0 while True: next_ballot_number = int(util.readfrom(util.root("nexttoprocess.txt"))) if count_to_process == 0: # wait here until get_count_to_process returns # it will wait on input instruction from stdio processing_command = get_processing_command(next_ballot_number) if processing_command.startswith("+"): next_ballot_number += const.num_pages util.writeto(util.root("nexttoprocess.txt"), next_ballot_number) count_to_process = 1 if processing_command.startswith("="): next_ballot_number = int(processing_command[1:]) util.writeto(util.root("nexttoprocess.txt"), next_ballot_number) count_to_process = 1 if processing_command.startswith("S"): count_to_process = 1 if processing_command.startswith("0"): count_to_process = 0 # we're done when we get instructed to process 0 if count_to_process == 0: break count_to_process -= 1 try: # get number of next image, # clean up, in case... gc.collect() log.debug("Request for %d" % (next_ballot_number, )) unprocs = [ incomingn(next_ballot_number + m) for m in range(const.num_pages) ] log.info(unprocs) # we need all images for sheet to be available to process it for filename in unprocs: if not os.path.exists(filename): errmsg = "File %s not present or available!" % ( base(filename), ) log.info(errmsg) # if a file is not yet available, that's not fatal raise FileNotPresentException(errmsg) #Processing #log.info("Processing %s:\n %s" % # (n, "\n".join("\t%s" % base(u) for u in unprocs)) #) log.debug("Creating ballot.") try: ballot = ballotfrom(unprocs, extensions) log.debug("Created ballot, processing.") results = ballot.ProcessPages() log.debug("Processed.") except BallotException as e: total_images_left_unprocessed += mark_error(e, *unprocs) log.exception("Could not process ballot") continue #Write all data #make dirs: proc1d = dirn("proc", next_ballot_number) resultsd = dirn("results", next_ballot_number) resultsfilename = filen(resultsd, next_ballot_number) for p in (proc1d, resultsd): util.mkdirp(p) #try: # results_to_vop_files(results,resultsfilename) #except Exception as e: # log.info(e) # print e #write csv and mosaic #log.info("local results_to_CSV") #csv = results_to_CSV(results,log) #log.info("Back from results_to_CSV") #util.genwriteto(resultsfilename + ".csv", csv) #write to the database try: log.debug("Inserting to db") dbc.insert(ballot) except db.DatabaseError: #dbc does not commit if there is an error, just need to remove #partial files remove_partial(resultsfilename + ".txt") remove_partial(resultsfilename + const.filename_extension) log.info("Could not commit to db") print "Could not commit to db!" util.fatal("Could not commit vote information to database") #Post-processing # move the images from unproc to proc log.debug("Renaming") procs = [ filen(proc1d, next_ballot_number + m) + const.filename_extension for m in range(const.num_pages) ] for a, b in zip(unprocs, procs): try: os.rename(a, b) except OSError as e: log.info("Could not rename %s" % a) util.fatal("Could not rename %s", a) total_images_processed += const.num_pages # Tell caller you've processed all images of this ballot log.debug("Requesting next") util.writeto(util.root("nexttoprocess.txt"), next_ballot_number + const.num_pages) # update next ballot file with next image number log.debug("Done writing nexttoprocess.txt") #print "%d extracted. " % (next_ballot_number,) log.info("%d images processed", const.num_pages) # for profiling # hp.heap().dump('prof.hpy');hp.setref();gc.collect(); # hp.setref();hp.heap().dump('prof.hpy') except FileNotPresentException, e: print e sys.stdout.flush()
def formatQuery(queryTemplateFileName, params): """ Read file at queries/@queryTemplateFileName and format it with @params using Django templates. """ return formatQueryFile(os.path.join(util.root(), "queries", queryTemplateFileName), params)
def main(): miss_counter = 0 # get command line arguments cfg_file = get_args() # read configuration from tevs.cfg and set constants for this run config.get(cfg_file) util.mkdirp(const.root) log = config.logger(const.logfilename) #create initial top level dirs, if they do not exist for p in ( "%s" % ("templates"), "%s%d" % ("template_images", os.getpid()), "%s%d" % ("composite_images", os.getpid()), "results", "proc", "errors"): util.mkdirp(util.root(p)) next_ballot = next.File(util.root("nexttoprocess.txt"), const.num_pages) try: ballotfrom = Ballot.LoadBallotType(const.layout_brand) except KeyError as e: util.fatal("No such ballot type: " + const.layout_brand + ": check " + cfg_file) # allow all instances to share a common template location, # though need per-pid locs for template_images and composite_images cache = Ballot.TemplateCache(util.root("templates")) extensions = Ballot.Extensions(template_cache=cache) # connect to db and open cursor if const.use_db: try: dbc = db.PostgresDB(const.dbname, const.dbuser) except db.DatabaseError: util.fatal("Could not connect to database") else: dbc = db.NullDB() total_proc, total_unproc = 0, 0 base = os.path.basename # While ballot images exist in the directory specified in tevs.cfg, # create ballot from images, get landmarks, get layout code, get votes. # Write votes to database and results directory. Repeat. #from guppy import hpy;hp=hpy();hp.setref();import gc;gc.disable();gc.collect();hp.setref() try: for n in next_ballot: gc.collect() unprocs = [incomingn(n + m) for m in range(const.num_pages)] if not os.path.exists(unprocs[0]): miss_counter += 1 log.info(base(unprocs[0]) + " does not exist. No more records to process") if miss_counter > 10: break continue #for i, f in enumerate(unprocs[1:]): # if not os.path.exists(f): # log.info(base(f) + " does not exist. Cannot proceed.") # for j in range(i): # log.info(base(unprocs[j]) + " will NOT be processed") # total_unproc += mark_error(None, *unprocs[:i-1]) #Processing log.info("Processing %s:\n %s" % (n, "\n".join("\t%s" % base(u) for u in unprocs)) ) try: ballot = ballotfrom(unprocs, extensions) results = ballot.ProcessPages() except BallotException as e: total_unproc += mark_error(e, *unprocs) log.exception("Could not process ballot") continue csv = Ballot.results_to_CSV(results) #moz = Ballot.results_to_mosaic(results) #Write all data #make dirs: proc1d = dirn("proc", n) resultsd = dirn("results", n) resultsfilename = filen(resultsd, n) for p in (proc1d, resultsd): util.mkdirp(p) try: results_to_vop_files(results,resultsfilename) except Exception as e: print e #write csv and mosaic util.genwriteto(resultsfilename + ".txt", csv) #write to the database try: dbc.insert(ballot) except db.DatabaseError: #dbc does not commit if there is an error, just need to remove #partial files remove_partial(resultsfilename + ".txt") remove_partial(resultsfilename + const.filename_extension) util.fatal("Could not commit vote information to database") #Post-processing # move the images from unproc to proc procs = [filen(proc1d, n + m) + const.filename_extension for m in range(const.num_pages)] for a, b in zip(unprocs, procs): try: os.rename(a, b) except OSError as e: util.fatal("Could not rename %s", a) total_proc += const.num_pages log.info("%d images processed", const.num_pages) #hp.heap().dump('prof.hpy');hp.setref();gc.collect();hp.setref();hp.heap().dump('prof.hpy') finally: cache.save_all() dbc.close() next_ballot.save() log.info("%d images processed", total_proc) if total_unproc > 0: log.warning("%d images NOT processed.", total_unproc)