def test_get_numbers_kpu(self): json_file_yosua = open('./resources/probabilities/kpu-probabilities.json') json_data_yosua = json.load(json_file_yosua) json_file_yosua.close() config_file = open('../static/datasets/digit_config_ppwp_scan_halaman_2_2019.json') config = json.load(config_file) outcomes = pp.get_possible_outcomes_for_config(config, json_data_yosua["probabilities"], 11, print_outcome_parsable) presidential = outcomes[0][0]['numbers'] self.assertEqual(presidential[0]['number'], 10) self.assertEqual(presidential[0]['shortName'], 'jokowi') self.assertEqual(presidential[1]['number'], 178) self.assertEqual(presidential[1]['shortName'], 'prabowo') self.assertEqual(presidential[2]['number'], 188) self.assertEqual(presidential[2]['shortName'], 'jumlah') summary = outcomes[1][0]['numbers'] self.assertEqual(summary[0]['number'], 188) self.assertEqual(summary[0]['shortName'], 'jumlah') self.assertEqual(summary[1]['number'], 10) self.assertEqual(summary[1]['shortName'], 'tidakSah') self.assertEqual(summary[2]['number'], 198) self.assertEqual(summary[2]['shortName'], 'jumlahSeluruh') self.assertAlmostEqual(outcomes[1][0]['confidence'], 0.11396216028446514) config_file.close()
def test_get_numbers2(self): json_file = open('./resources/probabilities/IMG_4221.json') json_data = json.load(json_file) config_file = open('../static/datasets/pilpres_2019_plano.json') config = json.load(config_file) json_file.close() numbers = json_data["numbers"] probabilities = [] for number in numbers: probability_set = {"id": number["id"]} number_probabilities = [] for digit_probability in number["extracted"]: number_probabilities.append(digit_probability["probabilities"]) probability_set["probabilitiesForNumber"] = number_probabilities probabilities.insert(0, probability_set) print("probz", probabilities) outcomes = pp.get_possible_outcomes_for_config(config, probabilities, 11) most_likely = outcomes[0][0] print("most likely", str(most_likely)) self.assertAlmostEqual(most_likely['confidence'], 0.7020011959926853) self.assertEqual(most_likely['jokowi'], 186) self.assertEqual(most_likely['prabowo'], 117) self.assertEqual(most_likely['jumlah'], 303)
def test_get_numbers_yosua(self): json_file_yosua = open('./resources/probabilities/yosua-probablities.json') json_data_yosua = json.load(json_file_yosua) json_file_yosua.close() outcomes = pp.get_possible_outcomes_for_config(self.config, json_data_yosua["probabilities"], 11, print_outcome_parsable) presidential = outcomes[0][0]['numbers'] self.assertEqual(presidential[0]['number'], 118) self.assertEqual(presidential[0]['shortName'], 'jokowi') self.assertEqual(presidential[1]['number'], 125) self.assertEqual(presidential[1]['shortName'], 'prabowo') self.assertEqual(presidential[2]['number'], 243) self.assertEqual(presidential[2]['shortName'], 'jumlah') summary = outcomes[1][0]['numbers'] self.assertEqual(summary[0]['number'], 243) self.assertEqual(summary[0]['shortName'], 'jumlah') self.assertEqual(summary[1]['number'], 7) self.assertEqual(summary[1]['shortName'], 'tidakSah') self.assertEqual(summary[2]['number'], 250) self.assertEqual(summary[2]['shortName'], 'jumlahSeluruh') self.assertAlmostEqual(outcomes[1][0]['confidence'], 0.7333740994255918)
def test_get_numbers(self): outcomes = pp.get_possible_outcomes_for_config(self.config, self.json_data["numbers"], 11, print_outcome) most_likely = outcomes[0][0] print("most likely", str(most_likely)) self.assertAlmostEqual(most_likely['confidence'], 0.7020011959926853) self.assertEqual(most_likely['jokowi'], 186) self.assertEqual(most_likely['prabowo'], 117) self.assertEqual(most_likely['jumlah'], 303)
def test_get_numbers(self): outcomes = pp.get_possible_outcomes_for_config( self.config, self.json_data["numbers"], 11, print_outcome) most_likely = outcomes[0][0] print("most likely", str(most_likely)) self.assertAlmostEqual(most_likely['confidence'], 0.7020011959926853) self.assertEqual(most_likely['jokowi'], 186) self.assertEqual(most_likely['prabowo'], 117) self.assertEqual(most_likely['jumlah'], 303)
def get_probabilities_result_parsable(request): json_data = json.loads(request.body.decode('utf-8')) outcomes = processprobs.get_possible_outcomes_for_config( load_config(json_data["configFile"]), json_data["probabilities"], settings.CATEGORIES_COUNT, print_outcome_parsable) results = [] for outcome in outcomes: results.append(outcome) output = {'probabilityMatrix': outcomes} return JsonResponse(output)
def test_get_numbers_yosua(self): json_file_yosua = open( './resources/probabilities/yosua-probablities.json') json_data_yosua = json.load(json_file_yosua) json_file_yosua.close() outcomes = pp.get_possible_outcomes_for_config( self.config, json_data_yosua["probabilities"], 11, print_outcome) most_likely = outcomes[0][0] print("most likely", str(most_likely)) self.assertAlmostEqual(most_likely['confidence'], 0.7020011959926853) self.assertEqual(most_likely['jokowi'], 186) self.assertEqual(most_likely['prabowo'], 117) self.assertEqual(most_likely['jumlah'], 303)
def test_get_halaman_tiga_numbers(self): json_file = open( './resources/probabilities/hal3_processprobs.wsgi.json') hal3_json_data = json.load(json_file) config_file = open('../static/datasets/hal_3_digit_config.json') hal3_config = json.load(config_file) outcomes = pp.get_possible_outcomes_for_config( hal3_config, hal3_json_data["probabilities"], 11) most_likely = outcomes[0] print(str(most_likely)) self.assertEqual(most_likely['ADPTLaki'], 221) self.assertEqual(most_likely['ADPTPerempuan'], 255) self.assertEqual(most_likely['ADPTJumlah'], 476) self.assertAlmostEqual(most_likely['confidence'], 0.699716392401)
def process_form(config_name, posted_config, scan_url): ref_form = lazy_load_reference_form(posted_config["referenceForm"]) with open(path.join(settings.STATIC_DIR, 'datasets/' + config_name), "w") as outfile: json.dump(posted_config, outfile, separators=(',', ':'), indent=4) before_dot = scan_url.find('.jpg') - 1 url_as_list = list(scan_url) try: start_page = int(url_as_list[before_dot]) pages_in_order = range(1, 6) pages_in_order.remove(start_page) pages_in_order.insert(0, start_page) except ValueError: pages_in_order = [] transform_output = None for page in pages_in_order: url_as_list[before_dot] = str(page) url_to_download = ''.join(url_as_list) download_file(url_to_download, "upload") try: registered_file = registration.process_file( None, 1, settings.STATIC_DIR, path.basename(url_to_download), ref_form, config_name) except Exception as err: logging.warning(Exception, err) continue transform_output = json.loads(registered_file) if transform_output["success"]: break if transform_output is None: download_file(scan_url, "upload") registered_file = registration.process_file(None, 1, settings.STATIC_DIR, path.basename(scan_url), ref_form, config_name) transform_output = json.loads(registered_file) transformed_url = transform_output["transformedUrl"] extracted = json.loads( extraction.extract("transformed/" + transformed_url, settings.STATIC_DIR, path.join(settings.STATIC_DIR, 'extracted'), settings.STATIC_DIR, posted_config)) digit_area = extracted["digitArea"] numbers = extracted["numbers"] probabilities = [] for number in numbers: probability_set = {"id": number["id"]} number_probabilities = [] for digit_probability in number["extracted"]: number_probabilities.append(digit_probability["probabilities"]) probability_set["probabilitiesForNumber"] = number_probabilities probabilities.insert(0, probability_set) outcomes = processprobs.get_possible_outcomes_for_config( posted_config, probabilities, settings.CATEGORIES_COUNT, print_outcome) results = [] for outcome in outcomes: results.append(outcome) output = json.dumps( { 'probabilityMatrix': outcomes, 'digitArea': digit_area }, separators=(',', ':')) return output
def download(request, kelurahan, tps, filename): config_file = "digit_config_pilpres_2019.json" loaded_config = load_config(config_file) try: maybe_extract_digits = json.loads( request.GET.get('extractDigits', 'true').lower()) calculate_numbers = json.loads( request.GET.get('calculateNumbers', 'true').lower()) store_files = json.loads( request.GET.get('storeFiles', 'false').lower()) a = __do_alignment(filename, kelurahan, request, tps, get_reference_form(config_file)) extract_digits = maybe_extract_digits or calculate_numbers start_lap = datetime.now() lap = datetime.now() b = extraction.extract(a['transformedUri'], settings.STATIC_DIR, path.join(settings.STATIC_DIR, 'extracted'), settings.STATIC_DIR, loaded_config, store_files) if extract_digits else { "numbers": [] } logging.info("2: Extract %s", (datetime.now() - lap).total_seconds()) lap = datetime.now() probabilities = [] for number in b["numbers"]: probability_set = {"id": number["id"]} number_probabilities = [] for digit_probability in number["extracted"]: number_probabilities.append(digit_probability["probabilities"]) probability_set["probabilitiesForNumber"] = number_probabilities probabilities.insert(0, probability_set) c = processprobs.get_possible_outcomes_for_config( loaded_config, probabilities, settings.CATEGORIES_COUNT, print_outcome) if calculate_numbers else {} logging.info("3: Probs %s", (datetime.now() - lap).total_seconds()) if calculate_numbers: del b['numbers'] del b['signatures'] b['probabilityMatrix'] = c output = {**a, **b} outcome = get_outcome(output) output['outcome'] = outcome if not store_files: io.storage.delete("static") del output['configFile'] del output['probabilityMatrix'] del output['digitArea'] del output['transformedUri'] del output['transformedUrl'] output['success'] = bool(outcome['confidence'] > 0.8) output['duration'] = (datetime.now() - start_lap).total_seconds() output['configFile'] = config_file except Exception as e: logging.exception( "failed 'download/<int:kelurahan>/<int:tps>/<str:filename>'") output = {'transformedUrl': None, 'success': False} return JsonResponse(output)