def test_get_numbers_kpu(self):
json_file_yosua = open('./resources/probabilities/kpu-probabilities.json')
json_data_yosua = json.load(json_file_yosua)
json_file_yosua.close()
config_file = open('../static/datasets/digit_config_ppwp_scan_halaman_2_2019.json')
config = json.load(config_file)
outcomes = pp.get_possible_outcomes_for_config(config, json_data_yosua["probabilities"], 11, print_outcome_parsable)
presidential = outcomes[0][0]['numbers']
self.assertEqual(presidential[0]['number'], 10)
self.assertEqual(presidential[0]['shortName'], 'jokowi')
self.assertEqual(presidential[1]['number'], 178)
self.assertEqual(presidential[1]['shortName'], 'prabowo')
self.assertEqual(presidential[2]['number'], 188)
self.assertEqual(presidential[2]['shortName'], 'jumlah')
summary = outcomes[1][0]['numbers']
self.assertEqual(summary[0]['number'], 188)
self.assertEqual(summary[0]['shortName'], 'jumlah')
self.assertEqual(summary[1]['number'], 10)
self.assertEqual(summary[1]['shortName'], 'tidakSah')
self.assertEqual(summary[2]['number'], 198)
self.assertEqual(summary[2]['shortName'], 'jumlahSeluruh')
self.assertAlmostEqual(outcomes[1][0]['confidence'], 0.11396216028446514)
config_file.close()
def test_get_numbers2(self):
json_file = open('./resources/probabilities/IMG_4221.json')
json_data = json.load(json_file)
config_file = open('../static/datasets/pilpres_2019_plano.json')
config = json.load(config_file)
json_file.close()
numbers = json_data["numbers"]
probabilities = []
for number in numbers:
probability_set = {"id": number["id"]}
number_probabilities = []
for digit_probability in number["extracted"]:
number_probabilities.append(digit_probability["probabilities"])
probability_set["probabilitiesForNumber"] = number_probabilities
probabilities.insert(0, probability_set)
print("probz", probabilities)
outcomes = pp.get_possible_outcomes_for_config(config, probabilities,
11)
most_likely = outcomes[0][0]
print("most likely", str(most_likely))
self.assertAlmostEqual(most_likely['confidence'], 0.7020011959926853)
self.assertEqual(most_likely['jokowi'], 186)
self.assertEqual(most_likely['prabowo'], 117)
self.assertEqual(most_likely['jumlah'], 303)
def test_get_numbers_yosua(self):
json_file_yosua = open('./resources/probabilities/yosua-probablities.json')
json_data_yosua = json.load(json_file_yosua)
json_file_yosua.close()
outcomes = pp.get_possible_outcomes_for_config(self.config, json_data_yosua["probabilities"], 11, print_outcome_parsable)
presidential = outcomes[0][0]['numbers']
self.assertEqual(presidential[0]['number'], 118)
self.assertEqual(presidential[0]['shortName'], 'jokowi')
self.assertEqual(presidential[1]['number'], 125)
self.assertEqual(presidential[1]['shortName'], 'prabowo')
self.assertEqual(presidential[2]['number'], 243)
self.assertEqual(presidential[2]['shortName'], 'jumlah')
summary = outcomes[1][0]['numbers']
self.assertEqual(summary[0]['number'], 243)
self.assertEqual(summary[0]['shortName'], 'jumlah')
self.assertEqual(summary[1]['number'], 7)
self.assertEqual(summary[1]['shortName'], 'tidakSah')
self.assertEqual(summary[2]['number'], 250)
self.assertEqual(summary[2]['shortName'], 'jumlahSeluruh')
self.assertAlmostEqual(outcomes[1][0]['confidence'], 0.7333740994255918)
def test_get_numbers(self):
outcomes = pp.get_possible_outcomes_for_config(self.config, self.json_data["numbers"], 11, print_outcome)
most_likely = outcomes[0][0]
print("most likely", str(most_likely))
self.assertAlmostEqual(most_likely['confidence'], 0.7020011959926853)
self.assertEqual(most_likely['jokowi'], 186)
self.assertEqual(most_likely['prabowo'], 117)
self.assertEqual(most_likely['jumlah'], 303)
def test_get_numbers(self):
outcomes = pp.get_possible_outcomes_for_config(
self.config, self.json_data["numbers"], 11, print_outcome)
most_likely = outcomes[0][0]
print("most likely", str(most_likely))
self.assertAlmostEqual(most_likely['confidence'], 0.7020011959926853)
self.assertEqual(most_likely['jokowi'], 186)
self.assertEqual(most_likely['prabowo'], 117)
self.assertEqual(most_likely['jumlah'], 303)
def get_probabilities_result_parsable(request):
json_data = json.loads(request.body.decode('utf-8'))
outcomes = processprobs.get_possible_outcomes_for_config(
load_config(json_data["configFile"]), json_data["probabilities"],
settings.CATEGORIES_COUNT, print_outcome_parsable)
results = []
for outcome in outcomes:
results.append(outcome)
output = {'probabilityMatrix': outcomes}
return JsonResponse(output)
def test_get_numbers_yosua(self):
json_file_yosua = open(
'./resources/probabilities/yosua-probablities.json')
json_data_yosua = json.load(json_file_yosua)
json_file_yosua.close()
outcomes = pp.get_possible_outcomes_for_config(
self.config, json_data_yosua["probabilities"], 11, print_outcome)
most_likely = outcomes[0][0]
print("most likely", str(most_likely))
self.assertAlmostEqual(most_likely['confidence'], 0.7020011959926853)
self.assertEqual(most_likely['jokowi'], 186)
self.assertEqual(most_likely['prabowo'], 117)
self.assertEqual(most_likely['jumlah'], 303)
def test_get_halaman_tiga_numbers(self):
json_file = open(
'./resources/probabilities/hal3_processprobs.wsgi.json')
hal3_json_data = json.load(json_file)
config_file = open('../static/datasets/hal_3_digit_config.json')
hal3_config = json.load(config_file)
outcomes = pp.get_possible_outcomes_for_config(
hal3_config, hal3_json_data["probabilities"], 11)
most_likely = outcomes[0]
print(str(most_likely))
self.assertEqual(most_likely['ADPTLaki'], 221)
self.assertEqual(most_likely['ADPTPerempuan'], 255)
self.assertEqual(most_likely['ADPTJumlah'], 476)
self.assertAlmostEqual(most_likely['confidence'], 0.699716392401)
def process_form(config_name, posted_config, scan_url):
ref_form = lazy_load_reference_form(posted_config["referenceForm"])
with open(path.join(settings.STATIC_DIR, 'datasets/' + config_name),
"w") as outfile:
json.dump(posted_config, outfile, separators=(',', ':'), indent=4)
before_dot = scan_url.find('.jpg') - 1
url_as_list = list(scan_url)
try:
start_page = int(url_as_list[before_dot])
pages_in_order = range(1, 6)
pages_in_order.remove(start_page)
pages_in_order.insert(0, start_page)
except ValueError:
pages_in_order = []
transform_output = None
for page in pages_in_order:
url_as_list[before_dot] = str(page)
url_to_download = ''.join(url_as_list)
download_file(url_to_download, "upload")
try:
registered_file = registration.process_file(
None, 1, settings.STATIC_DIR, path.basename(url_to_download),
ref_form, config_name)
except Exception as err:
logging.warning(Exception, err)
continue
transform_output = json.loads(registered_file)
if transform_output["success"]:
break
if transform_output is None:
download_file(scan_url, "upload")
registered_file = registration.process_file(None, 1,
settings.STATIC_DIR,
path.basename(scan_url),
ref_form, config_name)
transform_output = json.loads(registered_file)
transformed_url = transform_output["transformedUrl"]
extracted = json.loads(
extraction.extract("transformed/" + transformed_url,
settings.STATIC_DIR,
path.join(settings.STATIC_DIR, 'extracted'),
settings.STATIC_DIR, posted_config))
digit_area = extracted["digitArea"]
numbers = extracted["numbers"]
probabilities = []
for number in numbers:
probability_set = {"id": number["id"]}
number_probabilities = []
for digit_probability in number["extracted"]:
number_probabilities.append(digit_probability["probabilities"])
probability_set["probabilitiesForNumber"] = number_probabilities
probabilities.insert(0, probability_set)
outcomes = processprobs.get_possible_outcomes_for_config(
posted_config, probabilities, settings.CATEGORIES_COUNT, print_outcome)
results = []
for outcome in outcomes:
results.append(outcome)
output = json.dumps(
{
'probabilityMatrix': outcomes,
'digitArea': digit_area
},
separators=(',', ':'))
return output
def download(request, kelurahan, tps, filename):
config_file = "digit_config_pilpres_2019.json"
loaded_config = load_config(config_file)
try:
maybe_extract_digits = json.loads(
request.GET.get('extractDigits', 'true').lower())
calculate_numbers = json.loads(
request.GET.get('calculateNumbers', 'true').lower())
store_files = json.loads(
request.GET.get('storeFiles', 'false').lower())
a = __do_alignment(filename, kelurahan, request, tps,
get_reference_form(config_file))
extract_digits = maybe_extract_digits or calculate_numbers
start_lap = datetime.now()
lap = datetime.now()
b = extraction.extract(a['transformedUri'], settings.STATIC_DIR,
path.join(settings.STATIC_DIR, 'extracted'),
settings.STATIC_DIR, loaded_config,
store_files) if extract_digits else {
"numbers": []
}
logging.info("2: Extract %s", (datetime.now() - lap).total_seconds())
lap = datetime.now()
probabilities = []
for number in b["numbers"]:
probability_set = {"id": number["id"]}
number_probabilities = []
for digit_probability in number["extracted"]:
number_probabilities.append(digit_probability["probabilities"])
probability_set["probabilitiesForNumber"] = number_probabilities
probabilities.insert(0, probability_set)
c = processprobs.get_possible_outcomes_for_config(
loaded_config, probabilities, settings.CATEGORIES_COUNT,
print_outcome) if calculate_numbers else {}
logging.info("3: Probs %s", (datetime.now() - lap).total_seconds())
if calculate_numbers:
del b['numbers']
del b['signatures']
b['probabilityMatrix'] = c
output = {**a, **b}
outcome = get_outcome(output)
output['outcome'] = outcome
if not store_files:
io.storage.delete("static")
del output['configFile']
del output['probabilityMatrix']
del output['digitArea']
del output['transformedUri']
del output['transformedUrl']
output['success'] = bool(outcome['confidence'] > 0.8)
output['duration'] = (datetime.now() - start_lap).total_seconds()
output['configFile'] = config_file
except Exception as e:
logging.exception(
"failed 'download/<int:kelurahan>/<int:tps>/<str:filename>'")
output = {'transformedUrl': None, 'success': False}
return JsonResponse(output)
