def index():
"""
Gets the calltimes from the history json file and displays the webpage with
the histories in list form.
"""
call_times = utils.file_to_json("call_history.json")['history']
return render_template("index.html", call_times=call_times)
def save(config_file, election_results):
'''
Saves generated data to a file for later processing. This merges existing manually extracted
data with generated data.
'''
nj_pdf_data = utils.file_to_json(config_file)
nj_pdf_data.update(election_results)
utils.json_to_file(config_file, nj_pdf_data)
def manual_load():
nj_pdf_data_manual = utils.file_to_json('config/nj_pdf_data_manual.json')
manual_pdfs = [
'2015-GenAssembly-gen-elect-results-1st-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-2nd-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-3rd-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-4th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-5th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-6th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-7th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-8th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-9th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-10th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-11th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-12th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-13th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-14th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-15th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-16th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-17th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-18th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-19th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-20th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-21st-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-22nd-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-23rd-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-24th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-25th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-26th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-27th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-28th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-29th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-30th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-31st-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-32nd-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-33rd-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-34th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-35th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-36th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-37th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-38th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-39th-ld-municipality.pdf',
'2015-GenAssembly-gen-elect-results-40th-ld-municipality.pdf',
]
def build_election_result(acc, pdf):
config = utils.get_config(pdf)
rows = nj_pdf_data_manual[pdf]
election = e.from_pdf([], config)
election['rows'] = rows
acc[pdf] = election
return acc
return reduce(build_election_result, manual_pdfs, {})
def mapping(project, mapping_set_id, ids):
mapping_def = file_to_json(f'{RESOURCES_DIR}/mappings/mapping.json')
mapping_obj = {
'name': MAPPING_NAME,
'definition': {
'mapping': mapping_def,
'entities': ids,
},
'revision': '1',
'mappingset': mapping_set_id,
'project': project,
}
return client.mappings.create(data=mapping_obj)
def generate():
'''
Creates data structure from extracted NJ PDFs
'''
pdfs = utils.file_to_json('config/pdf_meta.json')
def build_election_result(acc, pdf):
config = utils.get_config(pdf)
rows = pe.extract_pdf_text('resources/tmp/{}'.format(pdf))
election = e.from_pdf(rows, config)
acc[pdf] = election
return acc
return reduce(build_election_result, pdfs.keys(), {})
def schema():
schema_names = []
schemas = []
defs = file_to_json(f'{RESOURCES_DIR}/schemas/all.json')
for obj in defs:
name = obj.get('name')
schema_names.append(name)
schema_obj = {
'name': name,
'type': 'record',
'revision': '1',
'definition': obj
}
schemas.append(schema_obj)
out = []
for obj in schemas:
try:
out.append(client.schemas.create(data=obj))
except Exception as err:
LOGGER.error(err)
LOGGER.error(obj)
return out
return reduce(build_election_result, manual_pdfs, {})
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-g',
'--generate',
action='store_true',
help='Generate nj_pdf_data.json file')
parser.add_argument('-s',
'--save',
action='store_true',
help='Save nj_pdf_data.json data to database')
args = parser.parse_args()
if args.generate:
generated = generate()
manually_loaded = manual_load()
generated.update(manually_loaded)
save('config/nj_pdf_data.json', generated)
elif args.save:
elections = utils.file_to_json('config/nj_pdf_data.json')
for _, election in elections.iteritems():
er.save(candidates=election['candidates'],
db=db,
parties=election['parties'],
election=election['election'],
rows=election['rows'])
sys.exit()
manager.types[person].override_property('occupant_age', MockFn(PERSON.get_age))
manager.types[person].override_property('occupant_gender', MockFn(PERSON.get_sex))
manager.types[person].override_property('occupant_name', MockFn(PERSON.get_name))
for _ in range(SEED_ENTITIES):
manager.register(person)
for mocker in manager.types.values():
if mocker.killed:
manager.kill()
return
break
manager.kill()
if __name__ == '__main__':
check_reqs(reqs=['KERNEL_URL', 'KERNEL_USER', 'KERNEL_PASSWORD'])
NAMES = file_to_json(f'{RESOURCES_DIR}/sample-names.json')
POP_CENTERS = file_to_json(f'{RESOURCES_DIR}/sample-locations.json')
args = sys.argv
seed = 1000
try:
if len(args) > 1 and isinstance(int(args[1]), int):
seed = int(args[1])
except ValueError:
pass
main(seed_size=seed)
