class VisualizationHandler(object):
def __init__(self):
self.os = os.name.lower()
config_handler = ConfigurationHandler(first_init=False)
self.config = config_handler.get_config()
self.cpr = ConditionalPrint(self.config.PRINT_MSA_HANDLER,
self.config.PRINT_EXCEPTION_LEVEL,
self.config.PRINT_WARNING_LEVEL)
def show_file_comparison_pycharm(self, filepath_1, filepath_2):
if self.os == 'linux' or self.os == 'posix':
try:
process = Popen(["charm", "diff", filepath_1, filepath_2])
return process
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
else:
self.cpr.printex("Write code here for other os, or take other os")
return None
def show_file_comparison_meld(self,
filepath_1,
filepath_2,
just_add_tab=False):
if self.os == 'linux' or self.os == 'posix':
try:
if just_add_tab:
process = Popen(
["meld", "--newtab", filepath_1, filepath_2])
else:
process = Popen(["meld", filepath_1, filepath_2])
return process
except Exception as ex:
self.cpr.printex("Exception calling meld", ex)
else:
self.cpr.printex("Write code here for other os, or take other os")
return None
class OCRVoter(object):
def __init__(self):
config_handler = ConfigurationHandler(first_init=False)
self.config = config_handler.get_config()
self.cpr = ConditionalPrint(self.config.PRINT_MSA_HANDLER,
self.config.PRINT_EXCEPTION_LEVEL,
self.config.PRINT_WARNING_LEVEL)
self.cpr_vocab_check = ConditionalPrint(
self.config.PRINT_VOCABULARY_CHECKER,
self.config.PRINT_EXCEPTION_LEVEL, self.config.PRINT_WARNING_LEVEL)
self.cpr_sc_predict = ConditionalPrint(
self.config.PRINT_SPECIALCHAR_PREDICTOR,
self.config.PRINT_EXCEPTION_LEVEL, self.config.PRINT_WARNING_LEVEL)
self.filo_last_chars = Filo(250)
self.predictor = None
self.use_aufsichtsrat_prediction = False
self.vocab_checker = None
self.previous_word_with_seperator = False
def add_predictor(self, predictor):
self.predictor = predictor
def add_vocab_checker(self, vocab_checker):
self.vocab_checker = vocab_checker
def get_same_count(self, c1, c2, c3):
same_ctr = 0
if c1 == c2:
same_ctr += 1
if c1 == c3:
same_ctr += 1
return same_ctr
def get_confidence_count(self,
char1,
char2,
char3,
cconf1,
cconf2,
cconf3,
wildcard_char='¦'):
def get_other_char(char_first, char_sec, char_thrd, co1, co2, co3):
if char_first != char_sec:
return char_sec, float(co2)
elif char_first != char_thrd:
return char_thrd, float(co3)
same_ctr = 0
cconf_ctr = float(cconf1)
if char1 == char2:
same_ctr += 1
cconf_ctr += float(cconf2)
if char1 == char3:
same_ctr += 1
cconf_ctr += float(cconf3)
# special cases space: ' ', ' ', 'x'
# wildcard character : '¦', '¦', '¦'
if char1 == ' ' and same_ctr == 1:
# if the confidence of the other character is below that value, space gets the high put in confidence value
return 1, 95.0 #todo j4t
SPACE_TRESH = 50.0
SPACE_PUT_IN_VALUE = 99.0
otherchar, otherconf = get_other_char(char1, char2, char3, cconf1,
cconf2, cconf3)
#print("otherchar",otherchar,"otherconf",otherconf)
if otherconf < SPACE_TRESH:
return 1, SPACE_PUT_IN_VALUE
elif char1 == wildcard_char and same_ctr == 1: #todo: differentiate type of character ??
# if there is two wildcards and one characters, characters confidence has to be higher than
# WILDCARD_TRESH to be taken
wildcard_tresh = 98.5
if self.config.MSA_BEST_CHANGE_VOTING_TRESHS_ON_EMPTY_LINE:
wildcard_tresh -= 10 # 0:99,19%, 20:99.16%, 10:99.27%
return 1, wildcard_tresh
elif char1 == wildcard_char and same_ctr == 0:
pass # todo maybe cover this case (cause wildcard has no confidence i.e if the two otherchars are very low prob, take wildcard)
elif char1 == '' and same_ctr == 0:
pass # todo maybe cover this case (cause space has no confidence ...
elif self.config.MSA_BEST_VOTING_DOWNSCALE_ONLY_SC \
and Random.is_special_character(char1) and same_ctr == 0 \
and char2 == wildcard_char and char3 == wildcard_char:
# lower the confidence of special characters which stand without any other chars
return same_ctr, cconf_ctr * 0.9
return same_ctr, cconf_ctr
def vote_best_of_three_simple(self,
text_1,
text_2,
text_3,
index_best,
wildcard_character='¦'):
list_line_1 = list(text_1)
list_line_2 = list(text_2)
list_line_3 = list(text_3)
accumulated_chars = ""
accumulated_confs = Filo
for character_index, character_1 in enumerate(list_line_1):
character_2 = list_line_2[character_index]
character_3 = list_line_3[character_index]
clist = [character_1, character_2, character_3]
# get the character which occurs the most
sc1 = self.get_same_count(character_1, character_2, character_3)
sc2 = self.get_same_count(character_2, character_1, character_3)
sc3 = self.get_same_count(character_3, character_2, character_1)
maxindices = np.argmax([sc2, sc1, sc3])
if maxindices == 0:
accumulated_chars += character_2
elif maxindices == 1:
accumulated_chars += character_1
else:
accumulated_chars += character_3
accumulated_chars_stripped = accumulated_chars.replace(
wildcard_character, '')
return accumulated_chars, accumulated_chars_stripped
def vote_best_of_three_charconfs(self,
line_1,
line_2,
line_3,
index_best,
wildcard_character='¦'):
try:
def try_obtain_charconf(value, undef_value=0):
if value is None or value is False or value is True:
return undef_value
return value
def try_obtain_char(charlist, index):
if index >= len(charlist):
return False #j4t means not defined
else:
return charlist[index]
key_confs_mapping = 'UID'
key_confs = 'x_confs'
key_char = 'calc_char'
self.cpr.print("vote_text1", line_1.textstr)
self.cpr.print("vote_text2", line_2.textstr)
self.cpr.print("vote_text3", line_3.textstr)
#if "¦¦lt.H" in line_1.textstr:
# self.cpr.print("asd")
maximum_char_number = max(len(line_1.textstr), len(line_2.textstr),
len(line_3.textstr))
accumulated_chars = ""
for character_index in range(
0, maximum_char_number
): # check: is list 1 always best reference?
character_1 = line_1.value(key_char, character_index)
character_2 = line_2.value(key_char, character_index)
character_3 = line_3.value(key_char, character_index)
charconf_1 = try_obtain_charconf(
line_1.value(key_confs, character_index, wsval=50.0))
charconf_2 = try_obtain_charconf(
line_2.value(key_confs, character_index, wsval=50.0))
charconf_3 = try_obtain_charconf(
line_3.value(key_confs, character_index, wsval=50.0))
clist = [character_1, character_2, character_3]
# get the character which occurs the most
sc1, acc_conf_1 = self.get_confidence_count(
character_1, character_2, character_3, charconf_1,
charconf_2, charconf_3)
sc2, acc_conf_2 = self.get_confidence_count(
character_2, character_1, character_3, charconf_2,
charconf_1, charconf_3)
sc3, acc_conf_3 = self.get_confidence_count(
character_3, character_2, character_1, charconf_3,
charconf_2, charconf_1)
maxindices = np.argmax([
acc_conf_2, acc_conf_1, acc_conf_3
]) # this takes in priorisation in case the chars are same
#todo:import to config
if character_index == maximum_char_number - 1 and character_2 == "¦" and character_3 == "¦" and character_1 == "I":
continue
if self.config.MSA_BEST_VOTER_DROP_CHARS_BELOW_TRESH == True:
tresh = self.config.MSA_BEST_VOTER_DROPPING_TRESH
maximum_conf = max(acc_conf_1, acc_conf_2, acc_conf_3)
if maximum_conf < tresh:
if [character_2, character_1, character_3
][maxindices] != '¦':
continue
if maxindices == 0:
accumulated_chars += character_2
elif maxindices == 1:
accumulated_chars += character_1
else:
accumulated_chars += character_3
accumulated_chars_stripped = accumulated_chars.replace(
wildcard_character, '')
return accumulated_chars, accumulated_chars_stripped
except Exception as ex:
tr = inspect.trace()
self.cpr.printex("ocr_voter.py Exception during confidence vote:",
ex)
self.cpr.printex("trace is:", tr)
def increase_umlaut_confidence(self, chars, charconfs):
charconfs_adapted = []
for char_index, char in enumerate(chars):
if char in SpecialChars.umlauts_caps or char in SpecialChars.umlauts:
cconf_to_add = charconfs[
char_index] + SpecialChars.umlaut_increment
elif char in SpecialChars.special_chars:
cconf_to_add = charconfs[
char_index] + SpecialChars.special_char_increment
else:
cconf_to_add = charconfs[char_index]
charconfs_adapted.append(cconf_to_add)
return charconfs_adapted
def vote_best_of_three_charconfs_searchspaces(self,
line_1,
line_2,
line_3,
index_best,
wildcard_character='¦'):
try:
key_confs_mapping = 'UID'
key_confs = 'x_confs'
key_char = 'calc_char'
self.cpr.print("vote_text1", line_1.textstr)
self.cpr.print("vote_text2", line_2.textstr)
self.cpr.print("vote_text3", line_3.textstr)
#if "Beteiligung:" in line_1.textstr:
# self.cpr.print("asd")
maximum_char_number = max(len(line_1.textstr), len(line_2.textstr),
len(line_3.textstr))
accumulated_chars = ""
accumulated_confs = Filo(300)
# search space settings
SEARCH_SPACE_Y_SIZE = 3
SEARCH_SPACE_X_SIZE_OUTER = 7
SEARCH_SPACE_X_SIZE_INNER = 3
SEARCH_SPACE_X_SEARCH_RANGE = 1
SEARCH_SPACE_PROCESSING_SUBSTITUTION_CHAR = '¦'
SEARCH_SPACE_PROCESSING_USE_SIMILAR_CHARS = True
SEARCH_RANGE = 1
PRINT_MATRICES = self.config.PRINT_SEARCH_SPACE_MATRICES
# initialize search space processor and search spaces
search_space_processor = SearchSpaceProcessor(SEARCH_SPACE_Y_SIZE, SEARCH_SPACE_X_SIZE_INNER, \
wildcard_character, SEARCH_SPACE_PROCESSING_SUBSTITUTION_CHAR)
ssp_chars = SearchSpace(SEARCH_SPACE_Y_SIZE,
SEARCH_SPACE_X_SIZE_OUTER,
SEARCH_SPACE_X_SEARCH_RANGE, True)
ssp_confs = SearchSpace(SEARCH_SPACE_Y_SIZE,
SEARCH_SPACE_X_SIZE_OUTER,
SEARCH_SPACE_X_SEARCH_RANGE, True)
# check if one of the lines is empty for certain settings
one_line_empty = False
if self.config.MSA_BEST_VOTER_PUSH_LESS_LINES_WHITESPACE_CONFS or \
self.config.MSA_BEST_CHANGE_VOTING_TRESHS_ON_EMPTY_LINE:
one_line_empty = self.check_if_one_line_empty(
[line_1, line_2, line_3], wildcard_character)
# loop through the maximum character range of the lines
range_extension = SEARCH_SPACE_X_SIZE_INNER
for character_index in range(
0, maximum_char_number + range_extension +
2): # check: is list 1 always best reference?
if character_index < maximum_char_number:
# if there is a character within range (no padding char from extension)
# get character values and obtain corresponding confidences (from searchspace because they might
# be different to normal values because of swapping
line_vals = [line_1.value(key_char, character_index), line_2.value(key_char, character_index), \
line_3.value(key_char, character_index)]
line_1_conf = line_1.value(key_confs,
character_index,
wsval=50.0)
line_2_conf = line_2.value(key_confs,
character_index,
wsval=50.0)
line_3_conf = line_3.value(key_confs,
character_index,
wsval=50.0)
charconf_1 = self.try_obtain_charconf_searchspace(
line_1_conf,
line_vals[0],
engine_key=line_1.name[0],
one_line_empty=one_line_empty)
charconf_2 = self.try_obtain_charconf_searchspace(
line_2_conf,
line_vals[1],
engine_key=line_2.name[0],
one_line_empty=one_line_empty)
charconf_3 = self.try_obtain_charconf_searchspace(
line_3_conf,
line_vals[2],
engine_key=line_3.name[0],
one_line_empty=one_line_empty)
charconf_vals = [charconf_1, charconf_2, charconf_3]
else:
# if the character is within padding range just give none values for characters and confidences
line_vals = [None, None, None]
charconf_vals = [None, None, None]
# fill searchspace with the chars and confidences
ssp_chars.push_column(line_vals)
ssp_confs.push_column(charconf_vals)
# update the mid-window of the search space (this is the actual search space processing step)
mid_chars = ssp_chars.get_middle_matrix(PRINT_MATRICES)
mid_confs = ssp_confs.get_middle_matrix(PRINT_MATRICES)
mid_chars_processed, mid_confs_processed, change_done = \
search_space_processor.process_search_space(mid_chars, mid_confs,SEARCH_SPACE_PROCESSING_USE_SIMILAR_CHARS)
if change_done is True:
ssp_chars.update_middle_matrix(mid_chars_processed)
ssp_confs.update_middle_matrix(mid_confs_processed)
# extract changed values from search space
character_offset = -(SEARCH_SPACE_X_SEARCH_RANGE + 1)
character_1 = ssp_chars.get_value_around_middle(
0, character_offset)
character_2 = ssp_chars.get_value_around_middle(
1, character_offset)
character_3 = ssp_chars.get_value_around_middle(
2, character_offset)
charconf_1 = ssp_confs.get_value_around_middle(
0, character_offset)
charconf_2 = ssp_confs.get_value_around_middle(
1, character_offset)
charconf_3 = ssp_confs.get_value_around_middle(
2, character_offset)
if character_1 is None or character_2 is None or character_3 is None:
# self.cpr.print("test")
continue
# in case umlaut confidence increment is active change charconfs otherwise same charconfs
charconf_1, charconf_2, charconf_3 = self.increase_umlaut_confidence_searchspace(
character_1, character_2, character_3, charconf_1,
charconf_2, charconf_3)
# get the previous characters from other lines as string (mainly for predictor)
filo_content = self.filo_last_chars.get_content_as_string()
# trigger predicted section for aufsichtsrat predictor
self.toggle_predictor(filo_content)
# predict_char if predictor is enabled
predicted_char = self.predict_char(filo_content)
# get the character which occurs the most by accumulating confidence scores
sc1, acc_conf_1 = self.get_confidence_count(
character_1, character_2, character_3, charconf_1,
charconf_2, charconf_3)
sc2, acc_conf_2 = self.get_confidence_count(
character_2, character_1, character_3, charconf_2,
charconf_1, charconf_3)
sc3, acc_conf_3 = self.get_confidence_count(
character_3, character_2, character_1, charconf_3,
charconf_2, charconf_1)
maxindices = np.argmax([
acc_conf_2, acc_conf_1, acc_conf_3
]) # this takes in priorisation in case the chars are same
if character_index == maximum_char_number + range_extension + 1 and character_2 == "¦" and character_3 == "¦" and character_1 == "I":
continue
# drop chars completely if they fall below a certain dropping treshhold and the setting is active
if self.config.MSA_BEST_VOTER_DROP_CHARS_BELOW_TRESH == True:
tresh = self.config.MSA_BEST_VOTER_DROPPING_TRESH
maximum_conf = max(acc_conf_1, acc_conf_2, acc_conf_3)
if maximum_conf < tresh:
if [character_2, character_1, character_3
][maxindices] != '¦':
continue
# determine character with the best accumulated confidence
voted_char = None
voted_acc_conf = None
if maxindices == 0:
voted_char = character_2
voted_acc_conf = acc_conf_2
elif maxindices == 1:
voted_char = character_1
voted_acc_conf = acc_conf_1
else:
voted_char = character_3
voted_acc_conf = acc_conf_3
# if predictor is active, check if there is a better char predicted which can replace voted character
voted_char = self.maybe_replace_voted_by_predicted_char(
voted_char, self.use_aufsichtsrat_prediction,
predicted_char, wildcard_character, voted_acc_conf,
character_1, character_2, character_3)
# push the voted char and the accumulated confidence of this char to results
accumulated_confs.push(voted_acc_conf)
accumulated_chars += voted_char
# if the predictor is enabled fill the filo with the voted_char
self.fill_filo_last_chars(voted_char)
# do vocabulary related steps, if activated
accumulated_chars = self.vocabulary_related_corrections(
accumulated_chars, wildcard_character, accumulated_confs)
# remove the wilcard characters and return result
accumulated_chars_stripped = accumulated_chars.replace(
wildcard_character, '')
return accumulated_chars, accumulated_chars_stripped
except Exception as ex:
tr = inspect.trace()
self.cpr.printex("ocr_voter.py Exception during confidence vote",
ex)
self.cpr.printex("trace", tr)
def vocabulary_related_corrections(self, accumulated_chars,
wildcard_character, accumulated_confs):
if self.config.KEYING_RESULT_VOCABULARY_CORRECTION_VOTE:
accumulated_chars_final = ""
acc_split = accumulated_chars.split()
len_split = len(acc_split)
for word_index, word in enumerate(acc_split):
if self.config.KEYING_RESULT_VC_IGNORE_SEPERATE_WRITING_CORRECTION:
if word_index == len_split - 1 and word.replace(
wildcard_character, "").endswith('-'):
self.previous_word_with_seperator = True
accumulated_chars_final += word + " "
continue
if word_index == 0:
if self.previous_word_with_seperator is True:
self.previous_word_with_seperator = False
accumulated_chars_final += word + " "
continue
acc_confs_word = accumulated_confs.pop_multi(len(word))
acc_conf, rate, change, word_starting_borders, word_trailing_borders, word_reduced = \
self.vocab_checker.get_accumulated_confidence_rate(word, acc_confs_word, wildcard_character)
self.cpr_vocab_check.print("w:", word, "wr:", word_reduced,
"accr:", acc_conf, "rate", rate)
# don't correct words below min vocab length ( mind that special chars in dict are toggled)
check_len = len(word)
if self.config.KEYING_RESULT_VC_DICT_REMOVE_SPECIAL_BORDER_CHARS:
check_len = len(word_reduced)
if check_len < self.config.KEYING_RESULT_VC_MIN_VOCAB_WORD_LENGTH:
accumulated_chars_final += word + " "
continue
if self.config.KEYING_RESULT_VC_CORRECT_ONLY_ERRONOUS_CHARS:
swappable_char_indices = []
acc_confs_used = None
word_used = None
if self.config.KEYING_RESULT_VC_CORRECT_ERRONOUS_SPECIAL_CHARS:
# use the full length confidences array including trailing and leading special characters
acc_confs_used = acc_confs_word
word_used = word
else:
# don't use trailing and starting special characters if no special chars needed
acc_confs_used = acc_confs_word[
len(word_starting_borders):(
len(acc_confs_word) -
len(word_trailing_borders))]
word_used = word_reduced
for conf_index, conf in enumerate(acc_confs_used):
if self.config.KEYING_RESULT_VC_CORRECT_ERRONOUS_SPECIAL_CHARS:
if conf <= 250:
character_related = word_used[conf_index]
is_special_char = Random.is_special_character(
character_related)
if is_special_char and character_related != wildcard_character:
# only swap special character indices
swappable_char_indices.append(conf_index)
else:
if conf <= 215:
swappable_char_indices.append(conf_index)
if len(swappable_char_indices) >= 1:
word_reduced_correct = self.vocab_checker.correct_text_at_certain_indices_only(
word_used, swappable_char_indices)
if word_reduced_correct != None:
word_correct_withtrails = None
if self.config.KEYING_RESULT_VC_CORRECT_ERRONOUS_SPECIAL_CHARS:
if Random.has_special_character(
word_reduced_correct):
# if special character was replaced with special character
word_correct_withtrails = word_reduced_correct
else:
# if special character was replaced by alphanumerical character
word_correct_withtrails = word
else:
word_correct_withtrails = word_starting_borders + word_reduced_correct + word_trailing_borders
# only print the changed results
if word != word_correct_withtrails:
self.cpr_vocab_check.print(
"w:", word, "wc:", word_correct_withtrails,
"accr:", acc_conf, "rate", rate)
accumulated_chars_final += word_correct_withtrails + " "
else:
accumulated_chars_final += word + " "
else:
accumulated_chars_final += word + " "
continue
if rate < self.config.KEYING_RESULT_VOCABULARY_CORRECTION_VOTE_TRESH \
and len(word_reduced) > 2:
# if the rate drops below tresh, try to fetch vocab entry
word_reduced_correct, suggestions, flh = self.vocab_checker.correct_text(
word_reduced)
if word_reduced_correct != None and word_reduced_correct != word_reduced:
word_correct_withtrails = word_starting_borders + word_reduced_correct + word_trailing_borders
self.cpr_vocab_check.print("w:", word, "wc:",
word_correct_withtrails,
"accr:", acc_conf, "rate",
rate)
accumulated_chars_final += word_correct_withtrails + " "
else:
accumulated_chars_final += word + " "
else:
accumulated_chars_final += word + " "
accumulated_chars = accumulated_chars_final
return accumulated_chars
def try_obtain_charconf_searchspace(
self,
value_confidence,
value,
undef_value=0,
engine_key=None,
one_line_empty=False,
):
if value_confidence is None or value_confidence is False or value_confidence is True:
return undef_value
returnvalue = value_confidence
if self.config.MSA_BEST_VOTER_SCALE_ENGINE_CONFIDENCES and engine_key is not None:
if engine_key == 'Abbyy':
if self.config.MSA_BEST_INCREASE_CONFIDENCE_OF_SOME_ABBYY_CHARS:
if value == "%": # improve ocropus in confidence of % because it was trained
value_confidence = value_confidence + 80
returnvalue = ConfidenceModifications.abby_factor * value_confidence
elif engine_key == 'Tess':
returnvalue = ConfidenceModifications.tesseract_factor * value_confidence
elif engine_key == 'Ocro':
returnvalue = ConfidenceModifications.ocropus_factor * value_confidence
if (self.config.MSA_BEST_VOTER_PUSH_LESS_LINES_WHITESPACE_CONFS and one_line_empty and value == " ") \
or (self.config.MSA_BEST_VOTER_PUSH_WHITESPACE_IF_MOSTLY_WILDCARD and one_line_empty \
and value == " "):
returnvalue += ConfidenceModifications.whitespace_push
return returnvalue
def check_if_one_line_empty(self, lines, wildcard_character):
for line in lines:
text_wo_wildcards = line.textstr.replace(wildcard_character, '')
if text_wo_wildcards == "":
return True
if self.config.MSA_BEST_VOTER_PUSH_WHITESPACE_IF_MOSTLY_WILDCARD:
# also count in high whitecard ratios as empty line
wildcard_ratio = 1 - (len(text_wo_wildcards) /
len(line.textstr))
if wildcard_ratio > 0.70:
return True
def toggle_predictor(self, filo_content):
if self.config.PREDICTOR_AUFSICHTSRAT_ENABLED:
if "Aufsichtsrat" in filo_content:
self.use_aufsichtsrat_prediction = True
if "Gründung:" in filo_content:
self.use_aufsichtsrat_prediction = False
def predict_char(self, filo_content):
predicted_char = None
if self.use_aufsichtsrat_prediction:
if len(filo_content
) >= 19: # if filo_content bigger than one prediction chunk
len_aufsichtsrat = 19
predicted_char = self.predictor.predict_next_aufsichtsrat_chars(
len_aufsichtsrat, filo_content)
# print("filo", filo_content,"predict:", predicted_char)
# print("dd")
return predicted_char
def fill_filo_last_chars(self, voted_char):
"""
fill filo for predictor usage with voted_char some additional chars around this char
:param voted_char:
:return:
"""
if self.config.PREDICTOR_AUFSICHTSRAT_ENABLED:
# create pre semi-tokenized input strings in the filos from the voted characters for prediction
if voted_char == ' ':
# the models usally use the 'ƿ' char in substitution for spaces
self.filo_last_chars.push(' ', filterchar='¦')
self.filo_last_chars.push('ƿ', filterchar='¦')
self.filo_last_chars.push(' ', filterchar='¦')
elif Random.is_special_character(voted_char):
self.filo_last_chars.push(' ', filterchar='¦')
self.filo_last_chars.push(voted_char, filterchar='¦')
self.filo_last_chars.push(' ', filterchar='¦')
else:
self.filo_last_chars.push(voted_char, filterchar='¦')
def increase_umlaut_confidence_searchspace(self, character_1, character_2,
character_3, charconf_1,
charconf_2, charconf_3):
if self.config.MSA_BEST_SEARCHSPACE_INCREASE_UMLAUT_CONFIDENCE:
clist = [character_1, character_2, character_3]
conflist = [charconf_1, charconf_2, charconf_3]
conflist_new = self.increase_umlaut_confidence(clist, conflist)
charconf_1 = conflist_new[0]
charconf_2 = conflist_new[1]
charconf_3 = conflist_new[2]
return charconf_1, charconf_2, charconf_3
return charconf_1, charconf_2, charconf_3
def maybe_replace_voted_by_predicted_char(self, voted_char, aufsichtsrat_prediction_toggled, predicted_char, \
wildcard_character, voted_acc_conf, character_1, character_2, character_3):
if aufsichtsrat_prediction_toggled:
if Random.is_special_character(predicted_char):
one_char_sc = Random.is_special_character(character_1) \
or Random.is_special_character(character_2) or Random.is_special_character(
character_3)
voted_char_sc = Random.is_special_character(voted_char)
if predicted_char != voted_char and (
one_char_sc
or voted_char_sc) and voted_char != wildcard_character:
# print("FiloContent:", filo_content)
self.cpr_sc_predict.print("pc:", predicted_char, "vc:",
voted_char, "vc_acc",
voted_acc_conf)
if voted_acc_conf <= 90.0:
if voted_char != '\f': # don't swap formfeeds, they don't get predicted at all
self.cpr_sc_predict.print("swap")
voted_char = predicted_char
return voted_char
class AdditionalInfoHandler(object):
def __init__(self):
config_handler = ConfigurationHandler(first_init=False)
self.config = config_handler.get_config()
self.cpr = ConditionalPrint(self.config.PRINT_ADDITIONAL_INFO_HANDLER,
self.config.PRINT_EXCEPTION_LEVEL,
self.config.PRINT_WARNING_LEVEL,
leading_tag=self.__class__.__name__)
self.cpr.print("init additional info handler")
def write_excel_to_json(self,
fileinfo,
filepath,
filetype,
idxcol=None,
parse_cols=None,
page=0):
""""
At the moment a little helper script for the Aktienführer-Project.
Be free to modify as you wish.
"""
#if isinstance(parse_cols, list): parse_cols = [parse_cols],
additional_filepath = path.normpath(
f"{filepath}/**/*{fileinfo.dbname}.{filetype}")
file = glob.glob(additional_filepath, recursive=True)
if len(file) != 1: return None
if filetype in ["xlsx", "xls"]:
df = pd.read_excel(file[0]).set_index("ProfileID")
jsondata = {fileinfo.dbname: {"Year": fileinfo.dbname}}
jsondf = df.to_dict(orient="index")
jsondata.update(jsondf)
with open(file[0].replace("xlsx", "json"), "w") as output:
json.dump(jsondata, output, indent=4)
return None
def fetch_additional_information_simple(self, file):
"""
Same as fetch additional information, but config related info is already included in given
parameters
:return: additional info
"""
if self.config.ADDITIONAL_INFORMATION:
additional_info = self.fetch_additional_information(
file,
self.config.INPUT_ADDINFOPATH,
idxcol=self.config.IDXCOL,
parse_cols=self.config.PARSE_COLS,
filetype=self.config.INPUT_ADDINFOFILETPYE)
return additional_info
return None
def fetch_additional_information(self,
fileinfo,
filepath,
filetype,
idxcol=None,
parse_cols=None,
page=0):
"""
Reads an additional file with information
It searches the file where the index_name matches tablename or dbname
:param file:
:param index_name:
:return: additional info
"""
#if isinstance(parse_cols, list): parse_cols = [parse_cols]
additional_filepath = path.normpath(
f"{filepath}/**/*{fileinfo.dbname}.{filetype}")
file = glob.glob(additional_filepath, recursive=True)
len_files = len(file)
if len_files > 1:
self.cpr.printex(
"More than one additional information file was found!")
return None
if len_files == 0:
self.cpr.printex("No additional information file was found!")
return None
file = file[0]
current_db_and_table = {
"db": fileinfo.dbname,
"table": fileinfo.tablename
}
if filetype in ["xlsx", "xls"]:
infos = {}
info_df = pd.read_excel(file) #.set_index("ProfileID")
parse_cols.remove(idxcol)
for db_and_table_id, current_db_and_tablename in current_db_and_table.items(
):
infos[db_and_table_id] = {}
for line, rubric_content in info_df.loc[
info_df[idxcol] ==
current_db_and_tablename][parse_cols].to_dict(
orient="index").items():
for rubric, content in rubric_content.items():
if rubric != idxcol:
if infos[db_and_table_id].get(rubric,
None) is None:
infos[db_and_table_id][rubric] = content
elif infos[db_and_table_id].get(rubric,
None) != content:
if not isinstance(
infos[db_and_table_id][rubric], list):
infos[db_and_table_id][rubric] = [
infos[db_and_table_id][rubric]
]
infos[db_and_table_id][rubric].append(content)
elif filetype == "json":
with open(file, "r") as add_info_file:
infos = json.load(add_info_file)
for possible_db_or_tablenames in reversed(list(infos.keys())):
possible_db_or_tablenames_orig = possible_db_or_tablenames # unchanged name
if self.config.ADD_INFO_SIMPLIFIED_NAME_COMPARISON:
psplit = possible_db_or_tablenames.split("-")
possible_db_or_tablenames = psplit[0]
if possible_db_or_tablenames not in current_db_and_table[
'table']:
del infos[possible_db_or_tablenames_orig]
else:
for db_and_table_id, current_db_and_tablename in current_db_and_table.items(
):
if possible_db_or_tablenames == current_db_and_tablename:
infos[db_and_table_id] = infos[
possible_db_or_tablenames_orig]
del infos[possible_db_or_tablenames_orig]
else:
return None
return infos
class IsriHandler(object):
def __init__(self):
self.os = os.name.lower()
config_handler = ConfigurationHandler(first_init=False)
self.config = config_handler.get_config()
if 'ExceptionInitializing' in self.config:
print("Exception initializing config, don't print")
self.cpr = ConditionalPrint(False, False, False)
else:
self.cpr = ConditionalPrint(self.config.PRINT_MSA_HANDLER,
self.config.PRINT_EXCEPTION_LEVEL,
self.config.PRINT_WARNING_LEVEL)
if self.os != 'linux' and self.os != 'posix':
raise OSError(
"Untested operating system adapt code and continue at own risk"
)
def accuracy(self,
path_correctfile,
path_generatedfile,
path_accuracy_report=""):
try:
call([
"accuracy", path_correctfile, path_generatedfile,
path_accuracy_report
])
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
class SynctextConfig(object):
def __init__(self):
self._used_config_acc = []
def use_T_algorithm(self):
self._used_config_acc.append("-T")
def use_H_algorithm(self):
self._used_config_acc.append("-H")
def use_case_insensitive(self):
self._used_config_acc.append("-i")
def use_display_suspect_markers_in_output(self):
self._used_config_acc.append("-s")
def get_used_config(self):
return self._used_config_acc
def clear_used_config(self):
self._used_config_acc = []
def synctext(self,
filepaths,
path_generatedfile=None,
synctext_config=SynctextConfig()):
try:
flags = synctext_config.get_used_config()
calls = ["synctext"]
calls.extend(flags)
calls.extend(filepaths)
if path_generatedfile is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_generatedfile, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def accsum(self, filepaths_accreports, path_generatedfile=None):
try:
calls = ["accsum"]
calls.extend(filepaths_accreports)
if path_generatedfile is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_generatedfile, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def groupacc(self,
path_groupfile,
path_accuracy_report,
path_groupacc_report=None):
try:
calls = ["groupacc"]
calls.append(path_groupfile)
calls.append(path_accuracy_report)
if path_groupacc_report is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_groupacc_report, True)
filehandle.close()
calls.append(path_groupacc_report)
call(calls)
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def accdist(self, filepaths_accreports, path_generated_xyfile=None):
try:
calls = ["accdist"]
calls.extend(filepaths_accreports)
if path_generated_xyfile is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_generated_xyfile, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
class NGramConfig(object):
def __init__(self):
self._used_config_acc = []
def set_ngram_size(self, number):
if number >= 1 and number <= 3:
self._used_config_acc.append("-n")
self._used_config_acc.append(str(number))
def clear_used_config(self):
self._used_config_acc = []
def get_used_config(self):
return self._used_config_acc
def ngram(self,
filepaths,
path_generatedfile=None,
ngram_config=NGramConfig()):
try:
flags = ngram_config.get_used_config()
calls = ["ngram"]
calls.extend(flags)
calls.extend(filepaths)
if path_generatedfile is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_generatedfile, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
class VoteConfig(object):
def __init__(self):
self._used_config_acc = []
def enable_O_optimization(self):
self._used_config_acc.append("-O")
def set_s(self, fraction_counter, fraction_denominator):
self._used_config_acc.append("-s")
self._used_config_acc.append(fraction_counter + "/" +
fraction_denominator)
def set_w(self, fraction_counter, fraction_denominator):
self._used_config_acc.append("-w")
self._used_config_acc.append(fraction_counter + "/" +
fraction_denominator)
def set_output_file(self, path_outputfile):
self._used_config_acc.append("-o")
self._used_config_acc.append(path_outputfile) #ok?
def clear_used_config(self):
self._used_config_acc = []
def get_used_config(self):
return self._used_config_acc
def vote(self, filepaths, ngram_config=VoteConfig()):
try:
flags = ngram_config.get_used_config()
calls = ["vote"]
calls.extend(flags)
calls.extend(filepaths)
call(calls)
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def wordacc(self,
path_correctfile,
path_comparison_file,
path_stopwordfile=None,
path_wordacc_report=None):
try:
calls = ["wordacc"]
if path_stopwordfile is not None:
calls.append("-S")
calls.append(path_stopwordfile)
calls.append(path_correctfile)
calls.append(path_comparison_file)
if path_wordacc_report is not None:
calls.append(path_wordacc_report)
call(calls)
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def wordaccsum(self, filepaths_wordacc_reports, path_accsumreport=None):
try:
calls = ["wordaccsum"]
calls.extend(filepaths_wordacc_reports)
if path_accsumreport is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_accsumreport, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def nonstopacc(self,
path_stopwordfile,
path_wordacc_report,
path_output_xyfile=None):
try:
calls = ["nonstopacc"]
calls.append(path_stopwordfile)
calls.append(path_wordacc_report)
if path_output_xyfile is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_output_xyfile, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def wordaccci(self, filepaths_wordacc_reports, path_outputfile=None):
try:
calls = ["wordaccci"]
calls.extend(filepaths_wordacc_reports)
if path_outputfile is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_outputfile, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def wordaccdist(self, filepaths_wordacc_reports, path_output_xyfile=None):
try:
calls = ["wordaccdist"]
calls.extend(filepaths_wordacc_reports)
if path_output_xyfile is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_output_xyfile, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def wordfreq(self, filepaths_inputtext, path_resultfile=None):
try:
calls = ["wordfreq"]
calls.extend(filepaths_inputtext)
if path_resultfile is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_resultfile, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
#todo add the zoning programs some day: point 4 in doc
def editop(self,
path_correctfile,
path_comparison_file,
path_editop_report=None):
try:
calls = ["editop"]
calls.append(path_correctfile)
calls.append(path_comparison_file)
if path_editop_report is not None:
calls.append(path_editop_report)
call(calls)
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def editopsum(self, filepaths_editopreports, path_summed_report=None):
try:
calls = ["editopsum"]
calls.extend(filepaths_editopreports)
if path_summed_report is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_summed_report, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def editopcost(self,
path_editop_report,
path_editop_report2=None,
path_output_xyfile=None):
try:
calls = ["editopcost"]
calls.append(path_editop_report)
if path_editop_report2 is not None:
calls.append(path_editop_report2)
if path_output_xyfile is None:
call(calls)
else:
filehandle = self.create_file_if_doesnt_exist(
path_output_xyfile, True)
call(calls, stdout=filehandle)
filehandle.close()
except Exception as ex:
self.cpr.printex("Exception calling pycharm", ex)
def create_file_if_doesnt_exist(self, filepath, overwrite=False):
file = open(filepath, 'w+')
if overwrite:
self.delete_file_content(file)
return file
def delete_file_content(self, pfile):
pfile.seek(0)
pfile.truncate()
class DictionaryHandler(object):
def __init__(self):
config_handler = ConfigurationHandler(first_init=False)
self.config = config_handler.get_config()
self.cpr = ConditionalPrint(self.config.PRINT_DICTIONARY_HANDLER,
self.config.PRINT_EXCEPTION_LEVEL,
self.config.PRINT_WARNING_LEVEL,
leading_tag=self.__class__.__name__)
self.cpr.print("init dictionary handler")
self.data_functs = None # storage for json object
self.data_titles = None # storage for json object
self.texts_functs = None
self.texts_titles = None
if self.config.USE_DICTIONARIES_FOR_PERSON_PARSING:
self.load_dictionaries()
# get the rows as sorted list of texts longest first
if self.data_functs is not None:
check_tf = self.sort_rows(self.get_rows(self.data_functs))
self.texts_functs = check_tf
if self.data_titles is not None:
check_tt = self.sort_rows(self.get_rows(self.data_titles))
self.texts_titles = check_tt
def diff_name_title(self, text_to_check):
len_text_to_check = len(text_to_check)
name_found = text_to_check
title_found = ""
for entry_index, entry in enumerate(self.texts_titles):
title, tlen = entry
# accelerate the process, by skipping comparisons which have longer texts
if tlen > len_text_to_check:
continue
# compare the texts
if title in text_to_check:
name_found = text_to_check.replace(title, "", 1).strip()
title_found = title
break
return name_found, title_found
def load_dictionaries(self):
base_dict_path = self.get_dict_path()
filepath_titles_dict = os.path.join(base_dict_path, "dict_titles.json")
filepath_functs_dict = os.path.join(base_dict_path, "dict_functs.json")
# load titles
if os.path.exists(filepath_titles_dict):
with open(filepath_titles_dict) as f:
self.data_titles = json.load(f)
else:
self.cpr.printex(
"dictionary dict_titles.json missing at specificied path",
filepath_titles_dict)
# load functs
if os.path.exists(filepath_functs_dict):
with open(filepath_functs_dict) as f:
self.data_functs = json.load(f)
else:
self.cpr.printex(
"dictionary dict_functs.json missing at specificied path",
filepath_functs_dict)
def get_rows(self, dict_data):
rows = dict_data['rows']
final_rows = []
for entry in rows:
text = entry[0]
final_rows.append((text, len(text)))
return final_rows
def sort_rows(self, rows):
#itemgetter(1),
rows.sort(key=lambda t: len(t[0]), reverse=True)
return rows
def path(self):
return os.getcwd()
def get_dict_path(self):
complete = os.path.join(self.path(), "additionals", "dictionaries")
return complete
class OCRset:
"""
A storage class for a y_mean value
and a set of lines which was assigned to each other
If the lineset values where not edited, they are intialized with 'False
"""
N_DISTANCE_SHORTEST_TAG = "n_distance_shortest"
def __init__(self, lines_size, y_mean, msa_handler):
lineset = []
for x in range(0, lines_size):
lineset.append(False)
self._set_lines = lineset
self._size = lines_size
self._y_mean = y_mean # mean y coordinate of all lines referenced in this set
self.shortest_distance_line_index = -1
self._unspaced = False # indicates the set_lines was unspaced
self._refspaced = False # indicates the set_lines was reference spaced
self._text_unspacer = TextUnspacer()
self.shortest_distance_line = None # holder element for recognized shortest distance line
self._best_msa_text = ""
self._text_seg = None
self._is_origin_database = False
self._database_handler = None
config_handler = ConfigurationHandler(first_init=False)
self._config = config_handler.get_config()
if 'ExceptionInitializing' in self._config:
print("Exception initializing config, don't print")
self._cpr = ConditionalPrint(False, False, False)
else:
self._cpr = ConditionalPrint(self._config.PRINT_MSA_HANDLER,
self._config.PRINT_EXCEPTION_LEVEL,
self._config.PRINT_WARNING_LEVEL)
self._msa_handler = msa_handler
def add_predictor(self, predictor):
self.predictor = predictor
self._msa_handler.add_predictor(predictor)
def is_database_set(self, enabled, database_handler):
self._is_origin_database = enabled
self._database_handler = database_handler
def edit_line_set_value(self, set_index, new_value):
self._set_lines[set_index] = new_value
def get_line_set_value_line(self, set_index):
return self._set_lines[set_index]
def get_line_set_value_text(self, set_index):
value_line = self.get_line_set_value_line(set_index)
value_text = self.get_line_content(value_line)
return value_text
def get_msa_best_text(self):
return self._best_msa_text
def set_msa_best_text(self, value):
self._best_msa_text = value
@property
def size(self):
return self._size
@size.setter
def size(self, value):
self._size = value
@property
def y_mean(self):
return self._y_mean
@y_mean.setter
def y_mean(self, value):
self.y_mean = value
def calculate_y_mean(self):
"""
Goes through set elements and calculates y_mean for y_start and y_stop values
:return:
"""
acc_counter = 0
y_start_final = 0
y_stop_final = 0
for line in self._set_lines:
# don't count undefined values for means
if line is False or line is None:
continue
# accumulate y-values
(x_start, y_start, x_stop, y_stop) = line.coordinates
y_start_final = y_start_final + y_start
y_stop_final = y_stop_final + y_stop
# add number of accumulation count
acc_counter = acc_counter + 1
y_start_mean = y_start_final / acc_counter
y_stop_mean = y_stop_final / acc_counter
y_mean = (y_start_mean + y_stop_mean) / 2
self._y_mean = round(y_mean)
def is_full(self):
"""
Checks if all lines are defined in the lineset
:return: True or False
"""
for line in self._set_lines:
if line is False:
return False
return True
def print_me(self, diff_only=False):
lineset_acc = ""
one_line_is_false = False
for line in self._set_lines:
try:
ocr_text = self.get_line_content(line)
if ocr_text is False:
one_line_is_false = True
lineset_acc = lineset_acc + str(ocr_text) + "||"
else:
lineset_acc = lineset_acc + ocr_text + "||"
except:
self._cpr.print("problem creating printable lineset ")
lineset_acc = lineset_acc + "||"
msa_str = str(self._best_msa_text)
if diff_only is True:
if one_line_is_false is True:
self._cpr.print(
str(self.y_mean) + "||" + msa_str + "||" +
str(self.shortest_distance_line_index) + "||" +
lineset_acc)
else:
self._cpr.print(
str(self.y_mean) + "||" + msa_str + "||" +
str(self.shortest_distance_line_index) + "||" + lineset_acc)
def calculate_n_distance_keying(self):
# get the texts
texts = []
for line in self._set_lines:
text = self.get_line_content(line)
texts.append(text)
self._n_distance_voter = NDistanceVoter(texts)
if "ExceptionInitializing" in self._config.keys():
print("Exception in initializing config using default in c")
shortest_dist_index = self._n_distance_voter.compare_texts( \
take_longest_on_empty_lines = True, \
vote_without_spaces = False)
else:
shortest_dist_index = self._n_distance_voter.compare_texts( \
take_longest_on_empty_lines = self._config.NDIST_VOTE_LONGEST_IF_EMPTY_STRINGS, \
vote_without_spaces = self._config.NDIST_VOTE_WITHOUT_SPACES)
# save the result
self.shortest_distance_line_index = shortest_dist_index
self.shortest_distance_line = self._set_lines[shortest_dist_index]
def calculate_n_distance_keying_wordwise(self):
if self._is_origin_database is False:
self._cpr.printex(
"Wordwise keying only possible with database originated ocr_sets"
)
raise Exception
# get maximum word index todo probably will be refactored
max_word_indices = []
for line in self._set_lines:
if line is False or line is None or line.textstr == '':
max_word_indices.append(0)
else:
max_word_index = int(max(line.data["word_idx"]))
max_word_indices.append(max_word_index)
max_word_index = max(max_word_indices)
self._cpr.print("mwi", max_word_index)
def get_word_at_calc_wordindex(line, word_index):
accumulated_word = ""
word_indices = line.data["calc_word_idx"]
for char_index, char in enumerate(line.data["char"]):
current_word_index = word_indices[char_index]
if current_word_index == word_index:
accumulated_word += char
if current_word_index > word_index:
break
return accumulated_word
max_word_index = 2
words_mock = [["hallo", "h4llo", "hallo"], ["zwei", None, "2wei"]]
ndist_voter = NDistanceVoter(None)
# get corresponding words
for current_word_index in range(0, max_word_index):
words = []
"""
for line in self._set_lines:
if line is False or line is None:
words.append(False)
else:
if current_word_index < int(max(line.data["calc_word_idx"])):
current_word = get_word_at_calc_wordindex(line, current_word_index)
words.append(current_word)
else:
words.append(False)
"""
words = words_mock[current_word_index]
ndist_voter.set_texts(words)
wordindex_result = ndist_voter.compare_texts( \
take_longest_on_empty_lines = self._config.NDIST_VOTE_LONGEST_IF_EMPTY_STRINGS, \
vote_without_spaces=self._config.NDIST_VOTE_WITHOUT_SPACES)
ndist_voter.reset()
self._cpr.print(words[wordindex_result])
self._cpr.print("--")
# just assume words is filled here and a 3 word list
return
def get_longest_index(self):
def if_notdef_set_emptystring(value):
if value is True or value is False or value is None:
return ""
return value
lsval_1 = if_notdef_set_emptystring(
self.get_line_content(self.get_line_set_value_line(0)))
lsval_2 = if_notdef_set_emptystring(
self.get_line_content(self.get_line_set_value_line(1)))
lsval_3 = if_notdef_set_emptystring(
self.get_line_content(self.get_line_set_value_line(2)))
len_pline_1 = len(lsval_1)
len_pline_2 = len(lsval_2)
len_pline_3 = len(lsval_3)
# max_index_value = max([len_pline_1, len_pline_2, len_pline_3])
max_index = np.argmax([len_pline_1, len_pline_2, len_pline_3])
self._cpr.print(max_index)
return max_index
def calculate_msa_best(self,
take_n_dist_best_index=False,
take_longest_as_pivot=False):
# do a preselection of best element, if the parameter is set to take best n_dist_index as a pivot
best_index = 1
if take_longest_as_pivot is True:
best_index = self.get_longest_index()
elif take_n_dist_best_index is True:
best_index = self.get_shortest_n_distance_index()
indices = [0, 1, 2]
indices.remove(best_index)
index1 = indices[0]
index2 = indices[1]
self._cpr.print("msa selection taking best:", best_index, "others:(",
index1, "and", index2, ")")
try:
line_1 = self.get_line_content(self._set_lines[index1])
line_2 = self.get_line_content(
self._set_lines[best_index]) # should be best
line_3 = self.get_line_content(self._set_lines[index2])
self._cpr.print("ocr_set:")
self._cpr.print("text_A", line_1)
self._cpr.print("text_B", line_2)
self._cpr.print("text_C", line_3)
lines = [line_1, line_2, line_3]
line_1_ok = not Random.is_false_true_or_none(line_1)
line_2_ok = not Random.is_false_true_or_none(line_2)
line_3_ok = not Random.is_false_true_or_none(line_3)
ok_lines = [line_1_ok, line_2_ok, line_3_ok]
not_ok_indices = []
ok_indices = []
for ok_index, ok in enumerate(ok_lines):
if ok is True:
# not_ok_indices.append(ok_index)
ok_indices.append(ok_index)
ok_len = len(ok_indices)
if ok_len == 1:
result = lines[ok_indices[0]]
elif ok_len == 0:
result = None
elif ok_len == 2:
result = lines[ok_indices[0]]
else:
result = self._msa_handler.get_best_of_three(
line_1, line_2, line_3)
self._best_msa_text = result
except Exception as e:
self._cpr.printex(
"ocr_set.py Exception in MSA, just taking line prio exception:",
e)
tr = inspect.trace()
self._cpr.printex("trace is:", tr)
self._best_msa_text = self.get_line_content(self._set_lines[1])
def obtain_best_index(self,
use_n_dist_pivot,
use_longest_pivot,
default_best_index=1):
# do a preselection of best element, if the parameter is set to take best n_dist_index as a pivot
best_index = 1
if use_n_dist_pivot is True:
ldist_best_index = self.get_shortest_n_distance_index(
) # this doesn't work in all cases atm
best_index = ldist_best_index
if use_longest_pivot is True:
best_index = self.get_longest_index()
indices = [0, 1, 2]
indices.remove(best_index)
other_indices = indices
return best_index, other_indices
def obtain_line_info(self, best_index, other_indices):
line_1 = self._set_lines[other_indices[0]]
line_2 = self._set_lines[best_index] # should be best
line_3 = self._set_lines[other_indices[1]]
text_1 = self.get_line_content(line_1)
text_2 = self.get_line_content(line_2) # should be best
text_3 = self.get_line_content(line_3)
self._cpr.print("ocr_set:")
self._cpr.print("text_A", text_1)
self._cpr.print("text_B", text_2)
self._cpr.print("text_C", text_3)
line_1_ok = not Random.is_false_true_or_none(line_1)
line_2_ok = not Random.is_false_true_or_none(line_2)
line_3_ok = not Random.is_false_true_or_none(line_3)
ok_lines = [line_1_ok, line_2_ok, line_3_ok]
ok_indices = []
for ok_index, ok in enumerate(ok_lines):
if ok is True:
# not_ok_indices.append(ok_index)
ok_indices.append(ok_index)
ok_len = len(ok_indices)
texts_return = [text_1, text_2, text_3]
lines_return = [line_1, line_2, line_3]
lines_return_ok = [line_1_ok, line_2_ok, line_3_ok]
return texts_return, lines_return, lines_return_ok, ok_len
def calculate_msa_best_all(self,
use_ndist_pivot,
use_longest_pivot,
use_charconfs,
use_wordwise,
use_searchspaces,
prefered_index=1):
# get the pivot index and the other indices
best_index, other_indices = self.obtain_best_index(
use_ndist_pivot, use_longest_pivot, prefered_index)
self._cpr.print("msa selection taking best:", best_index, "others:(",
other_indices[0], "and", other_indices[1], ")")
# fetch the lines to process and info which (and how many) lines are ok
texts, lines, lines_ok, number_lines_ok = self.obtain_line_info(
best_index, other_indices)
# do the msa if there is at least one line ok (confidence vote can be done with one line also :))
if use_wordwise is True:
if number_lines_ok != 0:
result, self._text_seg = self._msa_handler.get_best_of_three_wordwise(
lines[0], lines[1], lines[2], use_charconfs,
use_searchspaces)
else:
result = None
else:
if number_lines_ok != 0:
text_1 = self.get_line_content(lines[0])
text_2 = self.get_line_content(lines[1]) # should be best
text_3 = self.get_line_content(lines[2])
result = self._msa_handler.get_best_of_three(
text_1,
text_2,
text_3,
line_1=lines[0],
line_2=lines[1],
line_3=lines[2],
use_charconfs=use_charconfs,
use_searchspaces=use_searchspaces)
else:
result = None
self._best_msa_text = result
def calculate_msa_best_charconf(self,
take_n_dist_best_index=False,
take_longest_as_pivot=True):
# do a preselection of best element, if the parameter is set to take best n_dist_index as a pivot
best_index = 1
if take_n_dist_best_index is True:
ldist_best_index = self.get_shortest_n_distance_index(
) # this doesn't work in all cases atm
best_index = ldist_best_index
if take_longest_as_pivot is True:
best_index = self.get_longest_index()
indices = [0, 1, 2]
indices.remove(best_index)
index1 = indices[0]
index2 = indices[1]
self._cpr.print("msa selection taking best:", best_index, "others:(",
index1, "and", index2, ")")
try:
line_1 = self._set_lines[index1]
line_2 = self._set_lines[best_index]
line_3 = self._set_lines[index2]
text_1 = self.get_line_content(line_1)
text_2 = self.get_line_content(line_2) # should be best
text_3 = self.get_line_content(line_3)
self._cpr.print("ocr_set:")
self._cpr.print("text_A", text_1)
self._cpr.print("text_B", text_2)
self._cpr.print("text_C", text_3)
lines = [text_1, text_2, text_3]
line_1_ok = not Random.is_false_true_or_none(text_1)
line_2_ok = not Random.is_false_true_or_none(text_2)
line_3_ok = not Random.is_false_true_or_none(text_3)
ok_lines = [line_1_ok, line_2_ok, line_3_ok]
not_ok_indices = []
ok_indices = []
for ok_index, ok in enumerate(ok_lines):
if ok is True:
# not_ok_indices.append(ok_index)
ok_indices.append(ok_index)
ok_len = len(ok_indices)
if ok_len == 0:
result = None
else:
result = self._msa_handler.get_best_of_three(text_1, text_2, text_3, use_charconfs=True, \
line_1=line_1,line_2=line_2,line_3=line_3)
self._best_msa_text = result
except Exception as e:
self._cpr.printex(
"ocr_set.py Exception in MSA, just taking line prio exception:",
e)
tr = inspect.trace()
self._cpr.printex("trace is:", tr)
if take_n_dist_best_index is True:
self._best_msa_text = self.get_line_content(
self._set_lines[ldist_best_index])
else:
self._best_msa_text = self.get_line_content(
self._set_lines[best_index])
def get_shortest_n_distance_text(self):
if self.shortest_distance_line_index >= 0:
line = self.shortest_distance_line
line_text = self.get_line_content(line)
return line_text
else:
return None
def set_shortest_n_distance_text(self, value):
if self.shortest_distance_line_index >= 0:
sd_line = self.shortest_distance_line
sd_line_new_value = self.set_line_content(sd_line, value)
self.set_shortest_n_distance_line(sd_line_new_value)
else:
return None
def get_shortest_n_distance_line(self):
if self.shortest_distance_line_index >= 0:
line = self.shortest_distance_line
return line
else:
return None
def set_shortest_n_distance_line(self, value):
self.shortest_distance_line = value
def get_shortest_n_distance_index(self):
if self.shortest_distance_line_index >= 0:
return self.shortest_distance_line_index
else:
return None
def print_shortest_n_distance_line(self):
line = self.get_shortest_n_distance_text()
if line is not None and line is not False:
self._cpr.print(line)
def print_msa_best_line(self):
msa_text = self._best_msa_text
if msa_text is not None and msa_text is not False:
print(msa_text)
else:
self._cpr.print(str(msa_text))
def get_line_content(self, line):
"""
Helper method to get line content, because ocropus content
has other access properties. Method behaves differently when
the current set is a database set
:param line: line element to check upn
:return: string with line content, or 'False if line isn't defined.
"""
# hint: the attribute checked is created by hocr_line_normalizer
if line is False:
return False
# elif hasattr(line, 'ocr_text_normalized'):
if self._is_origin_database is False:
# just the standard behaviour
if line.ocr_text_normalized is not None:
return line.ocr_text_normalized
else:
return line.ocr_text
else:
return line.textstr
def set_line_content(self, line, value):
"""
Helper method to set line content, because ocropus content
has other access properties.
:param line: line element to set the value to
:param value: value to set to 'ocr_text_normalized' property
:return: line or false if line not defined
"""
# hint: the attribute checked is created by hocr_line_normalizer
if line is False:
return False
line.ocr_text_normalized = value
return line
def unspace_lines(self, list_index_to_unspace, unspaced_list_index):
unspaced_lines = self._text_unspacer.unspace_texts(
self._set_lines, list_index_to_unspace, unspaced_list_index)
self._unspaced = True
self._refspaced = False
self._set_lines = unspaced_lines
def refspace_lines(self, list_index_to_adapt, list_index_reference):
refspaced_lines = self._text_unspacer.refspace_texts(
self._set_lines, list_index_to_adapt, list_index_reference)
self._unspaced = False
self._refspaced = True
self._set_lines = refspaced_lines