def main():
global log
cfg_file = "tevs.cfg"
out_file = open("summary.csv","w")
config.get(cfg_file)
log = config.logger(util.root("log.txt"))
#pdb.set_trace()
#matches = is_this_like_them("hello there",{"hellc there":"bonjour","yabba":"dabba"})
if not const.use_db:
print "The configuration file indicates no database is in use."
print "We will now build totals from the results files."
build_totals_from_results_files()
output_totals_from_results_files()
return 0
try:
dbc = db.PostgresDB(const.dbname, const.dbuser)
print "Generating totals from db %s, user %s" % (const.dbname,
const.dbuser)
print "If there are many vote ops, this may take several minutes."
print "The next output will be the number of vote opportunities."
qs = query_database(dbc)
except db.DatabaseError:
print "Although the configuration file indicates a database is in use,"
print "we could not connect for dbname %s user %s." % (const.dbname,
const.dbuser)
print "We will now build totals from the results files."
build_totals_from_results_files()
output_totals_from_results_files()
return 0
return 0
def main():
global log
cfg_file = "tevs.cfg"
out_file = open("summary.csv", "w")
config.get(cfg_file)
log = config.logger(util.root("log.txt"))
#pdb.set_trace()
#matches = is_this_like_them("hello there",{"hellc there":"bonjour","yabba":"dabba"})
if not const.use_db:
print "The configuration file indicates no database is in use."
print "We will now build totals from the results files."
build_totals_from_results_files()
output_totals_from_results_files()
return 0
try:
dbc = db.PostgresDB(const.dbname, const.dbuser)
print "Generating totals from db %s, user %s" % (const.dbname,
const.dbuser)
print "If there are many vote ops, this may take several minutes."
print "The next output will be the number of vote opportunities."
qs = query_database(dbc)
except db.DatabaseError:
print "Although the configuration file indicates a database is in use,"
print "we could not connect for dbname %s user %s." % (const.dbname,
const.dbuser)
print "We will now build totals from the results files."
build_totals_from_results_files()
output_totals_from_results_files()
return 0
return 0
def main(counter, duplex, comment, inches, resolution):
# read configuration from tevs.cfg and set constants for this run
const.debug = False #XXX
config.get("tevs.cfg")
util.mkdirp(const.root)
log = config.logger(util.root("scan.log"))
inches_to_mm = 25.4
inc = 1
if duplex:
inc = 2
num = next.IncrementingFile(util.root("nexttoscan.txt"), inc)
try:
scanner = Scanner(
duplex,
int(inches * inches_to_mm),
resolution
)
while True:
counter = num.value()
print "Scanning",counter
stamp = datetime.now().isoformat()
for i, img in enumerate(scanner.scan(counter)):
#get path
n = counter + i
p = "%03d" % (n/1000,)
f = "%06d" % n
dir = util.root(const.incoming, p)
util.mkdirp(dir)
filename = os.path.join(dir, f + ".jpg")
img.save(filename)
print "Saved",filename
log.info("Saved %s at %s\n%s", filename, stamp, comment)
num.increment_and_save()
except ScanningException:
print "Empty feeder?"
log.info("Scan aborted due to empty feeder for 20 seconds.")
sys.exit(2)
except KeyboardInterrupt:
log.info("Scan aborted by user")
sys.exit(1)
def mark_error(e, *files):
log = logging.getLogger('')
if e is not None:
log.error(e)
for file in files:
log.error("Could not process ballot " + os.path.basename(file))
try:
shutil.copy2(file, util.root("errors", os.path.basename(file)))
except IOError:
log.error("Could not copy unprocessable file to errors dir")
return len(files)
def mark_error(e, *files):
log = logging.getLogger('')
if e is not None:
log.error(e)
for file in files:
log.error("Could not process ballot " + os.path.basename(file))
try:
shutil.copy2(file, util.root("errors", os.path.basename(file)))
except IOError:
log.error("Could not copy unprocessable file to errors dir")
return len(files)
def main(counter, duplex, comment, inches, resolution):
# read configuration from tevs.cfg and set constants for this run
const.debug = False #XXX
config.get("tevs.cfg")
util.mkdirp(const.root)
log = config.logger(util.root("scan.log"))
inches_to_mm = 25.4
inc = 1
if duplex:
inc = 2
num = next.IncrementingFile(util.root("nexttoscan.txt"), inc)
try:
scanner = Scanner(duplex, int(inches * inches_to_mm), resolution)
while True:
counter = num.value()
print "Scanning", counter
stamp = datetime.now().isoformat()
for i, img in enumerate(scanner.scan(counter)):
#get path
n = counter + i
p = "%03d" % (n / 1000, )
f = "%06d" % n
dir = util.root(const.incoming, p)
util.mkdirp(dir)
filename = os.path.join(dir, f + ".jpg")
img.save(filename)
print "Saved", filename
log.info("Saved %s at %s\n%s", filename, stamp, comment)
num.increment_and_save()
except ScanningException:
print "Empty feeder?"
log.info("Scan aborted due to empty feeder for 20 seconds.")
sys.exit(2)
except KeyboardInterrupt:
log.info("Scan aborted by user")
sys.exit(1)
def main(counter, duplex, comment, inches, resolution):
# read configuration from tevs.cfg and set constants for this run
const.debug = False #XXX
config.get("tevs.cfg")
util.mkdirp(const.root)
log = config.logger(util.root("scan.log"))
inches_to_mm = 25.4
print counter
inc = 1
if duplex:
inc = 2
num = next.Simple(counter, inc)
if counter < 0:
num = next.File(util.root("nexttoscan.txt"), inc)
try:
scanner = Scanner(duplex, int(inches * inches_to_mm), resolution)
for counter in num:
stamp = datetime.now().isoformat()
for i, img in enumerate(scanner.scan(counter)):
#get path
n = counter + i
p = "%03d" % (n / 1000, )
f = "%06d" % n
dir = util.root(const.incoming, p)
util.mkdirp(dir)
file = os.path.join(dir, f + ".jpg")
img.save(file)
log.info("Saving %s at %s\n%s", file, stamp, comment)
num.save()
except KeyboardInterrupt:
log.info("Scan aborted by user")
sys.exit(1)
def executeQuery(sparqlEndpoint, query, returnFormat = JSON):
"""
Execute SPARQL @query on @sparqlEndpoint.
Cache the query results temporarily in the tmp directory.
"""
path = os.path.join(util.root(), "tmp", util.sha1(sparqlEndpoint.endpoint + query))
if os.path.exists(path):
with open(path, "rb") as file:
return pickle.load(file)
else:
sparqlEndpoint.setQuery(unicode(query.decode("UTF-8")))
sparqlEndpoint.setReturnFormat(returnFormat)
results = sparqlEndpoint.query().convert()
with open(path, "wb") as file:
pickle.dump(results, file)
return results
def main():
cfg_file = "tevs.cfg"
out_file = open("summary.csv", "w")
config.get(cfg_file)
log = config.logger(util.root("log.txt"))
if not const.use_db:
log.error("Cannot summarize database without a database")
return 1
try:
dbc = db.PostgresDB(const.dbname, const.dbuser)
print "preparing summary for", const.dbname, "user", const.dbuser
except db.DatabaseError:
log.error("Could not connect to database")
return 2
qs = query(dbc, out_file)
pdb.set_trace()
printer = display['console']
print printer(qs)
return 0
def __init__(self,
x1,
y1,
x2,
y2,
v_margin=1,
h_margin=1,
side="Noside",
layout_id="Nolayoutid",
image=None,
image_filename="Nofilename",
jurisdiction="Nojurisdiction",
contest="Nocontest",
choice="Nochoice",
max_votes=1,
logger=None):
self.x1 = x1
self.y1 = y1
self.x2 = x2
self.y2 = y2
logger.info("VOP at %d %d %d %d" %
(self.x1, self.y1, self.x2, self.y2))
self.crop_bbox = (x1 - h_margin, y1 - v_margin, x2 + h_margin,
y2 + v_margin)
# test crop_bbox for reasonableness, and recenter if necessary
# report recentered coordinates as adjusted_x and y.
self.image = image
self.image_filename = image_filename
self.side = side
self.layout_id = layout_id
self.jurisdiction = jurisdiction
self.contest = contest
self.choice = choice
self.histogram = None
self.red_mean = None
self.red_lowest = None
self.red_low = None
self.red_high = None
self.red_highest = None
self.green_mean = None
self.green_lowest = None
self.green_low = None
self.green_high = None
self.green_highest = None
self.blue_mean = None
self.blue_lowest = None
self.blue_low = None
self.blue_high = None
self.blue_highest = None
self.voted = None
self.ambiguous = None
self.max_votes = max_votes
self.logger = logger
# overvoted flag is set by BallotSideWalker
# when it has finished checking all votes for a box
self.overvoted = False
# vote test sets self.voted and self.ambiguous
# returns 0 on success but 1 if updating self coordinates
self.crop = image.crop(self.crop_bbox)
self.stat = ImageStat.Stat(self.crop)
# fills in r/g/b mean values
self.mean()
# make a copy of self.mean, and throw out the adjustment
# if this copy is darker than the new copy
(orig_red_mean, orig_green_mean, orig_blue_mean) = self.stat.mean
# !!!new mjt 11/12/13
ret_coord = coord_adjust.coord_adjust(self.crop, h_margin, v_margin,
x2 - x1, y2 - y1)
if ret_coord is not None:
#print ret_coord, "applies to old crop now saved."
#self.crop.save("/tmp/saved.jpg")
# the returned coordinate is the ULC in a crop box w/ margins
self.x1 = x1 + ret_coord[0] - h_margin
self.y1 = y1 + ret_coord[1] - v_margin
self.x2 = x2 + ret_coord[0] - h_margin
self.y2 = y2 + ret_coord[1] - v_margin
logger.info("VOP adj to %d %d %d %d" %
(self.x1, self.y1, self.x2, self.y2))
#print "VOP adj to %d %d %d %d" % (self.x1,self.y1,self.x2,self.y2)
self.crop_bbox = (self.x1 - h_margin, self.y1 - v_margin,
self.x2 + h_margin, self.y2 + v_margin)
self.crop = image.crop(self.crop_bbox)
#self.crop.save("/tmp/saved_adj.jpg")
#pdb.set_trace()
self.stat = ImageStat.Stat(self.crop)
# fills in r/g/b mean values
self.mean()
# fills in r/g/b lowest/low/high/highest values
# undo move if it turns out this red_mean is lighter than orig_red_mean
if orig_red_mean < self.red_mean:
self.x1 = x1
self.y1 = y1
self.x2 = x2
self.y2 = y2
self.crop_bbox = (x1 - h_margin, y1 - v_margin, x2 + h_margin,
y2 + v_margin)
self.crop = image.crop(self.crop_bbox)
self.stat = ImageStat.Stat(self.crop)
# fills in r/g/b mean values
self.mean()
logger.info("VOP unadj back to %d %d %d %d" %
(self.x1, self.y1, self.x2, self.y2))
self.hist4()
self.vote_test()
# save image around write-in
if (self.choice.find("Write") >= 0) and (self.voted or self.ambiguous):
#print "self.choice",self.choice
#print "self.voted",self.voted
#print "self.ambiguous",self.ambiguous
# we should save the image with the writein to our writeins folder
# using the contest text's first few chars,
# first, generate a subimage using the upper left
# of self.crop_bbox and extending for the contest width and
# for 1".
# !!!TODO
# image_to_save = self.image.crop((self.crop_bbox[0],
# self.crop_bbox[1],
# self.crop_bbox[2]+(the column's width),
# self.crop_bbox[3]+dpi))
outcropbox = None
try:
wizhop = const.writein_zone_horiz_offset_inches * const.dpi
#self.logger.info("Wizhop %s" % (wizhop,))
wizvop = const.writein_zone_vert_offset_inches * const.dpi
#self.logger.info("Wizvop %s" % (wizvop,))
wizwidth = const.writein_zone_width_inches * const.dpi
#self.logger.info("Wizwidth %s" % (wizwidth,))
wizheight = const.writein_zone_height_inches * const.dpi
#self.logger.info("Wizheight %s" % (wizheight,))
outfile = os.path.join(
util.root("writeins"),
self.contest[:4] + os.path.basename(image_filename))
outcropbox = (int(self.crop_bbox[0] + wizhop),
int(self.crop_bbox[1] + wizvop),
int(self.crop_bbox[0] + wizhop + wizwidth),
int(self.crop_bbox[1] + wizvop + wizheight))
self.image.crop(outcropbox).save(outfile)
self.logger.info("Wrote writein image %s" % (outfile, ))
except Exception as e:
self.logger.warning("Could not save writein image %s %s %s" %
(outfile, outcropbox, e))
def merge_with_composites(self):
"""update composite images for this layout with this ballot's images"""
try:
oldimage = Image.open("%s/%s%d/%s.jpg" % (
util.root(),
"composite_images",os.getpid(),
tmpl.barcode))
except:
try:
oldimage = Image.open("%s/%s%d/%s.jpg" % (
util.root(),
"template_images",os.getpid(),
tmpl.barcode))
except:
oldimage = None
# else retrieve template image
# derotate the new image as if you were building a template
r2d = 180/3.14
newimage = page.image.rotate(-r2d * page.rot, Image.BILINEAR)
#page.image.save("/tmp/postrotate.jpg")
if oldimage is None: oldimage = newimage
# landmarks will change once image is derotated!
try:
self.FindLandmarks(pagenum)
except BallotException:
pass
# translate the new image to align with composite image
delta_x = tmpl.xoff - page.xoff
delta_y = tmpl.yoff - page.yoff
newimage = newimage.offset(delta_x,delta_y)
#newimage.save("/tmp/posttranslate.jpg")
# apply darker operation, save result in first argument?
oldimage.load()
oldr,oldg,oldb = oldimage.split()
newr,newg,newb = newimage.split()
# count dark pixels in oldr excluding edges
oldr_crop = oldr.crop((const.dpi/4,
const.dpi/4,
min(oldr.size[0],newr.size[0])
- (const.dpi/4),
min(oldr.size[1],newr.size[1])
- (const.dpi/4)))
old_total_intensity = 0
for p in oldr_crop.getdata():
old_total_intensity += p
newr = ImageChops.darker(oldr,newr)
newg = ImageChops.darker(oldg,newg)
newb = ImageChops.darker(oldb,newb)
new_total_intensity = 0
# count dark pixels in newr excluding edges
newr_crop = newr.crop((const.dpi/4,
const.dpi/4,
min(newr.size[0],oldr.size[0])
- (const.dpi/4),
min(newr.size[1],oldr.size[1])
- (const.dpi/4)))
new_total_intensity = 0
for p in newr_crop.getdata():
new_total_intensity += p
self.logger.info("%s Old %d New %d Diff %d" % (
os.path.basename(page.filename),
old_total_intensity,
new_total_intensity,
old_total_intensity - new_total_intensity))
newimage = Image.merge("RGB",(newr,newg,newb))
try:
composite_counts_dict[tmpl.barcode] += 1
if 0==(composite_counts_dict[tmpl.barcode]%5):
self.logger.info(
"Composite count for %s now %d (this run only)" % (
tmpl.barcode,
composite_counts_dict[tmpl.barcode]
))
except:
composite_counts_dict[tmpl.barcode] = 1
newimage.save("%s/%s%d/%s.jpg" % (
util.root(),
"composite_images",os.getpid(),
tmpl.barcode))
def dirn(dir, n): # where dir is the "unrooted" name
return util.root(dir, _fn(n))
def main():
miss_counter = 0
# get command line arguments
cfg_file = get_args()
# read configuration from tevs.cfg and set constants for this run
config.get(cfg_file)
util.mkdirp(const.root)
log = config.logger(const.logfilename)
#create initial top level dirs, if they do not exist
for p in ("%s" % ("templates"), "%s%d" % ("template_images", os.getpid()),
"%s%d" % ("composite_images", os.getpid()), "results", "proc",
"errors"):
util.mkdirp(util.root(p))
next_ballot = next.File(util.root("nexttoprocess.txt"), const.num_pages)
try:
ballotfrom = Ballot.LoadBallotType(const.layout_brand)
except KeyError as e:
util.fatal("No such ballot type: " + const.layout_brand + ": check " +
cfg_file)
# allow all instances to share a common template location,
# though need per-pid locs for template_images and composite_images
cache = Ballot.TemplateCache(util.root("templates"))
extensions = Ballot.Extensions(template_cache=cache)
# connect to db and open cursor
if const.use_db:
try:
dbc = db.PostgresDB(const.dbname, const.dbuser)
except db.DatabaseError:
util.fatal("Could not connect to database")
else:
dbc = db.NullDB()
total_proc, total_unproc = 0, 0
base = os.path.basename
# While ballot images exist in the directory specified in tevs.cfg,
# create ballot from images, get landmarks, get layout code, get votes.
# Write votes to database and results directory. Repeat.
#from guppy import hpy;hp=hpy();hp.setref();import gc;gc.disable();gc.collect();hp.setref()
try:
for n in next_ballot:
gc.collect()
unprocs = [incomingn(n + m) for m in range(const.num_pages)]
if not os.path.exists(unprocs[0]):
miss_counter += 1
log.info(
base(unprocs[0]) +
" does not exist. No more records to process")
if miss_counter > 10:
break
continue
#for i, f in enumerate(unprocs[1:]):
# if not os.path.exists(f):
# log.info(base(f) + " does not exist. Cannot proceed.")
# for j in range(i):
# log.info(base(unprocs[j]) + " will NOT be processed")
# total_unproc += mark_error(None, *unprocs[:i-1])
#Processing
log.info("Processing %s:\n %s" % (n, "\n".join("\t%s" % base(u)
for u in unprocs)))
try:
ballot = ballotfrom(unprocs, extensions)
results = ballot.ProcessPages()
except BallotException as e:
total_unproc += mark_error(e, *unprocs)
log.exception("Could not process ballot")
continue
csv = Ballot.results_to_CSV(results)
#moz = Ballot.results_to_mosaic(results)
#Write all data
#make dirs:
proc1d = dirn("proc", n)
resultsd = dirn("results", n)
resultsfilename = filen(resultsd, n)
for p in (proc1d, resultsd):
util.mkdirp(p)
try:
results_to_vop_files(results, resultsfilename)
except Exception as e:
print e
#write csv and mosaic
util.genwriteto(resultsfilename + ".txt", csv)
#write to the database
try:
dbc.insert(ballot)
except db.DatabaseError:
#dbc does not commit if there is an error, just need to remove
#partial files
remove_partial(resultsfilename + ".txt")
remove_partial(resultsfilename + const.filename_extension)
util.fatal("Could not commit vote information to database")
#Post-processing
# move the images from unproc to proc
procs = [
filen(proc1d, n + m) + const.filename_extension
for m in range(const.num_pages)
]
for a, b in zip(unprocs, procs):
try:
os.rename(a, b)
except OSError as e:
util.fatal("Could not rename %s", a)
total_proc += const.num_pages
log.info("%d images processed", const.num_pages)
#hp.heap().dump('prof.hpy');hp.setref();gc.collect();hp.setref();hp.heap().dump('prof.hpy')
finally:
cache.save_all()
dbc.close()
next_ballot.save()
log.info("%d images processed", total_proc)
if total_unproc > 0:
log.warning("%d images NOT processed.", total_unproc)
def main():
NextEqualsPrefix = "Next="
MorePrompt = ":"
NextToProcessFile = ""
miss_counter = 0
# get command line arguments
cfg_file = get_args()
# read configuration from tevs.cfg and set constants for this run
config.get(cfg_file)
util.mkdirp(const.root)
log = config.logger(util.root("extraction.log"))
# create initial toplevel directories if they don't exist
for p in ("%s" % ("templates"), "%s" % ("template_images"),
"%s" % ("composite_images"), "results", "proc", "errors"):
util.mkdirp(util.root(p))
# make sure you have code for ballot type spec'd in config file
try:
ballotfrom = Ballot.LoadBallotType(const.layout_brand)
except KeyError as e:
util.fatal("No such ballot type: %s check %s !",
(const.layout_brand, cfg_file))
cache = Ballot.TemplateCache(util.root("templates"))
extensions = Ballot.Extensions(template_cache=cache)
# connect to db and open cursor
if const.use_db:
try:
dbc = db.PostgresDB(database=const.dbname, user=const.dbuser)
except db.DatabaseError:
util.fatal("Could not connect to database!")
else:
dbc = db.NullDB()
log.info("Database connected.")
total_images_processed, total_images_left_unprocessed = 0, 0
base = os.path.basename
# Each time given a signal to proceed for count_to_process ballots,
# create ballot from images, get landmarks, get layout code, get votes.
# Write votes to database and results directory.
# for profiling
# from guppy import hpy;hp=hpy();hp.setref();
# import gc;gc.disable();gc.collect();hp.setref()
NextToProcessFile = util.root("nexttoprocess.txt")
count_to_process = 0
file_problem = False
while True:
log.debug("Top of loop.")
next_ballot_number = int(util.readfrom(NextToProcessFile))
log.debug("Read %d from %s" % (next_ballot_number, NextToProcessFile))
if count_to_process == 0:
# send prompt to controlling process, "READY:" or "+ for SKIP:"
if file_problem:
file_problem = False
# do not remove space after %06d
print "Next=%06d , + to SKIP:" % (next_ballot_number, )
else:
# do not remove space after %06d
print "Next=%06d , READY:" % (next_ballot_number, )
sys.stdout.flush()
# wait here until get_count_to_process returns
# it will wait on input instruction from stdio
try:
count_to_process = get_count_to_process(
next_ballot_number, log)
except DoIncrementException, e:
log.debug("Do increment exception")
util.writeto(NextToProcessFile,
next_ballot_number + const.num_pages)
log.debug(
"Wrote %d to next_ballot_number, count to process is %d" %
(next_ballot_number + const.num_pages, count_to_process))
count_to_process = 0
log.debug("Setting count to process to 0.")
continue
# we're done when we get instructed to process 0
if count_to_process == 0:
break
count_to_process -= 1
try:
# get number of next image,
# clean up, in case...
gc.collect()
log.debug("Request for %d" % (next_ballot_number, ))
unprocs = [
incomingn(next_ballot_number + m)
for m in range(const.num_pages)
]
log.info(unprocs)
# we need all images for sheet to be available to process it
for filename in unprocs:
log.info("Checking for path.")
if not os.path.exists(filename):
log.info("File not present.")
errmsg = "File %s not present or available!!!" % (
base(filename), )
log.info(errmsg.replace("!!!", ""))
print errmsg
sys.stdout.flush()
raise FileNotPresentException(filename)
log.info("Path found.")
#Processing
log.debug("Creating ballot.")
try:
ballot = ballotfrom(unprocs, extensions)
log.debug("Created ballot, processing.")
results = ballot.ProcessPages()
log.debug("Processed.")
except BallotException as e:
total_images_left_unprocessed += mark_error(e, *unprocs)
log.exception("Could not process ballot")
util.writeto(NextToProcessFile,
next_ballot_number + const.num_pages)
continue
#Write all data
#make dirs:
proc1d = dirn("proc", next_ballot_number)
resultsd = dirn("results", next_ballot_number)
resultsfilename = filen(resultsd, next_ballot_number)
for p in (proc1d, resultsd):
util.mkdirp(p)
#try:
# results_to_vop_files(results,resultsfilename)
#except Exception as e:
# log.info(e)
# print e
#write csv and mosaic
#log.info("local results_to_CSV")
#csv = results_to_CSV(results,log)
#log.info("Back from results_to_CSV")
#util.genwriteto(resultsfilename + ".csv", csv)
#write to the database
try:
log.debug("Inserting to db")
dbc.insert(ballot)
except db.DatabaseError:
#dbc does not commit if there is an error, just need to remove
#partial files
remove_partial(resultsfilename + ".txt")
remove_partial(resultsfilename + const.filename_extension)
log.info("Could not commit to db")
print "Could not commit to db!"
util.fatal("Could not commit vote information to database")
#Post-processing
# move the images from unproc to proc
log.debug("Renaming")
procs = [
filen(proc1d, next_ballot_number + m) +
const.filename_extension for m in range(const.num_pages)
]
for a, b in zip(unprocs, procs):
try:
os.rename(a, b)
except OSError as e:
log.info("Could not rename %s" % a)
util.fatal("Could not rename %s", a)
total_images_processed += const.num_pages
# Tell caller you've processed all images of this ballot
log.debug("Requesting next")
util.writeto(NextToProcessFile,
next_ballot_number + const.num_pages)
# update next ballot file with next image number
log.debug("Done writing nexttoprocess.txt")
#print "%d extracted. " % (next_ballot_number,)
log.info("%d images processed", const.num_pages)
# for profiling
# hp.heap().dump('prof.hpy');hp.setref();gc.collect();
# hp.setref();hp.heap().dump('prof.hpy')
except FileNotPresentException, e:
file_problem = True
print "FileNotPresentException"
sys.stdout.flush()
log.info("FileNotPresentException occurred")
continue
def main():
miss_counter = 0
# get command line arguments
cfg_file = get_args()
# read configuration from tevs.cfg and set constants for this run
config.get(cfg_file)
util.mkdirp(const.root)
log = config.logger(const.logfilename)
log.info("Log created.")
# create initial toplevel directories if they don't exist
for p in ("%s" % ("templates"), "%s" % ("template_images"),
"%s" % ("composite_images"), "results", "proc", "errors"):
util.mkdirp(util.root(p))
# make sure you have code for ballot type spec'd in config file
try:
ballotfrom = Ballot.LoadBallotType(const.layout_brand)
except KeyError as e:
util.fatal("No such ballot type: " + const.layout_brand + ": check " +
cfg_file)
cache = Ballot.TemplateCache(util.root("templates"))
extensions = Ballot.Extensions(template_cache=cache)
# connect to db and open cursor
if const.use_db:
try:
dbc = db.PostgresDB(database=const.dbname, user=const.dbuser)
except db.DatabaseError:
util.fatal("Could not connect to database")
else:
dbc = db.NullDB()
log.info("Database connected.")
total_images_processed, total_images_left_unprocessed = 0, 0
base = os.path.basename
# Each time given a signal to proceed for count_to_process ballots,
# create ballot from images, get landmarks, get layout code, get votes.
# Write votes to database and results directory.
# for profiling
# from guppy import hpy;hp=hpy();hp.setref();
# import gc;gc.disable();gc.collect();hp.setref()
count_to_process = 0
while True:
next_ballot_number = int(util.readfrom(util.root("nexttoprocess.txt")))
if count_to_process == 0:
# wait here until get_count_to_process returns
# it will wait on input instruction from stdio
processing_command = get_processing_command(next_ballot_number)
if processing_command.startswith("+"):
next_ballot_number += const.num_pages
util.writeto(util.root("nexttoprocess.txt"),
next_ballot_number)
count_to_process = 1
if processing_command.startswith("="):
next_ballot_number = int(processing_command[1:])
util.writeto(util.root("nexttoprocess.txt"),
next_ballot_number)
count_to_process = 1
if processing_command.startswith("S"):
count_to_process = 1
if processing_command.startswith("0"):
count_to_process = 0
# we're done when we get instructed to process 0
if count_to_process == 0:
break
count_to_process -= 1
try:
# get number of next image,
# clean up, in case...
gc.collect()
log.debug("Request for %d" % (next_ballot_number, ))
unprocs = [
incomingn(next_ballot_number + m)
for m in range(const.num_pages)
]
log.info(unprocs)
# we need all images for sheet to be available to process it
for filename in unprocs:
if not os.path.exists(filename):
errmsg = "File %s not present or available!" % (
base(filename), )
log.info(errmsg)
# if a file is not yet available, that's not fatal
raise FileNotPresentException(errmsg)
#Processing
#log.info("Processing %s:\n %s" %
# (n, "\n".join("\t%s" % base(u) for u in unprocs))
#)
log.debug("Creating ballot.")
try:
ballot = ballotfrom(unprocs, extensions)
log.debug("Created ballot, processing.")
results = ballot.ProcessPages()
log.debug("Processed.")
except BallotException as e:
total_images_left_unprocessed += mark_error(e, *unprocs)
log.exception("Could not process ballot")
continue
#Write all data
#make dirs:
proc1d = dirn("proc", next_ballot_number)
resultsd = dirn("results", next_ballot_number)
resultsfilename = filen(resultsd, next_ballot_number)
for p in (proc1d, resultsd):
util.mkdirp(p)
#try:
# results_to_vop_files(results,resultsfilename)
#except Exception as e:
# log.info(e)
# print e
#write csv and mosaic
#log.info("local results_to_CSV")
#csv = results_to_CSV(results,log)
#log.info("Back from results_to_CSV")
#util.genwriteto(resultsfilename + ".csv", csv)
#write to the database
try:
log.debug("Inserting to db")
dbc.insert(ballot)
except db.DatabaseError:
#dbc does not commit if there is an error, just need to remove
#partial files
remove_partial(resultsfilename + ".txt")
remove_partial(resultsfilename + const.filename_extension)
log.info("Could not commit to db")
print "Could not commit to db!"
util.fatal("Could not commit vote information to database")
#Post-processing
# move the images from unproc to proc
log.debug("Renaming")
procs = [
filen(proc1d, next_ballot_number + m) +
const.filename_extension for m in range(const.num_pages)
]
for a, b in zip(unprocs, procs):
try:
os.rename(a, b)
except OSError as e:
log.info("Could not rename %s" % a)
util.fatal("Could not rename %s", a)
total_images_processed += const.num_pages
# Tell caller you've processed all images of this ballot
log.debug("Requesting next")
util.writeto(util.root("nexttoprocess.txt"),
next_ballot_number + const.num_pages)
# update next ballot file with next image number
log.debug("Done writing nexttoprocess.txt")
#print "%d extracted. " % (next_ballot_number,)
log.info("%d images processed", const.num_pages)
# for profiling
# hp.heap().dump('prof.hpy');hp.setref();gc.collect();
# hp.setref();hp.heap().dump('prof.hpy')
except FileNotPresentException, e:
print e
sys.stdout.flush()
def formatQuery(queryTemplateFileName, params):
"""
Read file at queries/@queryTemplateFileName and format it with @params
using Django templates.
"""
return formatQueryFile(os.path.join(util.root(), "queries", queryTemplateFileName), params)
def main():
miss_counter = 0
# get command line arguments
cfg_file = get_args()
# read configuration from tevs.cfg and set constants for this run
config.get(cfg_file)
util.mkdirp(const.root)
log = config.logger(const.logfilename)
#create initial top level dirs, if they do not exist
for p in (
"%s" % ("templates"),
"%s%d" % ("template_images", os.getpid()),
"%s%d" % ("composite_images", os.getpid()),
"results",
"proc",
"errors"):
util.mkdirp(util.root(p))
next_ballot = next.File(util.root("nexttoprocess.txt"), const.num_pages)
try:
ballotfrom = Ballot.LoadBallotType(const.layout_brand)
except KeyError as e:
util.fatal("No such ballot type: " + const.layout_brand + ": check " + cfg_file)
# allow all instances to share a common template location,
# though need per-pid locs for template_images and composite_images
cache = Ballot.TemplateCache(util.root("templates"))
extensions = Ballot.Extensions(template_cache=cache)
# connect to db and open cursor
if const.use_db:
try:
dbc = db.PostgresDB(const.dbname, const.dbuser)
except db.DatabaseError:
util.fatal("Could not connect to database")
else:
dbc = db.NullDB()
total_proc, total_unproc = 0, 0
base = os.path.basename
# While ballot images exist in the directory specified in tevs.cfg,
# create ballot from images, get landmarks, get layout code, get votes.
# Write votes to database and results directory. Repeat.
#from guppy import hpy;hp=hpy();hp.setref();import gc;gc.disable();gc.collect();hp.setref()
try:
for n in next_ballot:
gc.collect()
unprocs = [incomingn(n + m) for m in range(const.num_pages)]
if not os.path.exists(unprocs[0]):
miss_counter += 1
log.info(base(unprocs[0]) + " does not exist. No more records to process")
if miss_counter > 10:
break
continue
#for i, f in enumerate(unprocs[1:]):
# if not os.path.exists(f):
# log.info(base(f) + " does not exist. Cannot proceed.")
# for j in range(i):
# log.info(base(unprocs[j]) + " will NOT be processed")
# total_unproc += mark_error(None, *unprocs[:i-1])
#Processing
log.info("Processing %s:\n %s" %
(n, "\n".join("\t%s" % base(u) for u in unprocs))
)
try:
ballot = ballotfrom(unprocs, extensions)
results = ballot.ProcessPages()
except BallotException as e:
total_unproc += mark_error(e, *unprocs)
log.exception("Could not process ballot")
continue
csv = Ballot.results_to_CSV(results)
#moz = Ballot.results_to_mosaic(results)
#Write all data
#make dirs:
proc1d = dirn("proc", n)
resultsd = dirn("results", n)
resultsfilename = filen(resultsd, n)
for p in (proc1d, resultsd):
util.mkdirp(p)
try:
results_to_vop_files(results,resultsfilename)
except Exception as e:
print e
#write csv and mosaic
util.genwriteto(resultsfilename + ".txt", csv)
#write to the database
try:
dbc.insert(ballot)
except db.DatabaseError:
#dbc does not commit if there is an error, just need to remove
#partial files
remove_partial(resultsfilename + ".txt")
remove_partial(resultsfilename + const.filename_extension)
util.fatal("Could not commit vote information to database")
#Post-processing
# move the images from unproc to proc
procs = [filen(proc1d, n + m) + const.filename_extension
for m in range(const.num_pages)]
for a, b in zip(unprocs, procs):
try:
os.rename(a, b)
except OSError as e:
util.fatal("Could not rename %s", a)
total_proc += const.num_pages
log.info("%d images processed", const.num_pages)
#hp.heap().dump('prof.hpy');hp.setref();gc.collect();hp.setref();hp.heap().dump('prof.hpy')
finally:
cache.save_all()
dbc.close()
next_ballot.save()
log.info("%d images processed", total_proc)
if total_unproc > 0:
log.warning("%d images NOT processed.", total_unproc)
def dirn(dir, n): # where dir is the "unrooted" name
return util.root(dir, _fn(n))
