def get_office_strings(self, sha256): js = self.retrieve_json_by_sha256(sha256) if js is None: return {'smart_parser_data_not_found': 1} rec = {'title': "", 'roles': [], 'departments': []} props = js.get('document_sheet_props', []) if len(props) > 0 and props[0].get('sheet_title') is not None: rec['title'] = normalize_whitespace(props[0]['sheet_title']) _, rec['title'] = TPredictionCase.truncate_title(rec['title']) roles = set() departments = set() for p in js.get('persons', []): role = p.get('person', {}).get('role') if role is not None and len(role) > 0 and len(roles) < 10: roles.add(normalize_whitespace(role)) department = p.get('person', {}).get('department') if department is not None and len(department) > 0 and len( departments) < 10: departments.add(normalize_whitespace(department)) rec['roles'] = list(roles) rec['departments'] = list(departments) return rec
def get_regions(inp, start_after_rf_line=True): regions = TRussianRegions() start = not start_after_rf_line used_regions = set() for line in inp: if not start: if line.find("Российская Федерация") != -1: start = True continue found_stop_word = False for stop_word in [ 'среднедушевые', 'рублей', '2016 год', 'i квартал', 'российская федерация', 'федеральный округ', 'наименование', 'в том числе', ' без ', '(кроме ', 'ямало-ненецкий автономный округ', 'предварительные', 'начиная' ]: if line.lower().find(stop_word) != -1: found_stop_word = True break if found_stop_word: continue cols = line.split("\t") region_str = cols[0].strip().replace('H', 'Н') region_str = normalize_whitespace(region_str) region = regions.get_region_in_nominative(region_str) if region is None: raise Exception("cannot find region {}".format(region_str)) used_regions.add(region.id) yield region, cols[1:] for r in regions.iterate_inner_regions_without_joined(): if r.id not in used_regions: raise Exception( "region {}, id={} is not found in the input file".format( r.name, r.id))
def convert_vehicle(name): name = unidecode(name) name = normalize_whitespace(name) name = name.replace(" ", "") name = name.replace("-", "") name = name.lower() return name
def get_region_in_nominative_and_dative(self, russian_name): russian_name = normalize_whitespace(russian_name.strip().lower()) region: TRegion for region in self.regions: for region_in_dative in region.dative_forms: if russian_name.endswith(region_in_dative): return self._region_id_to_region[region.id] return self.get_region_in_nominative(russian_name)
def get_region_all_forms(self, text, unknown_region=None): text = normalize_whitespace(text.strip().lower()) best_region_id = unknown_region max_form_len = 0 for pos, (region_id, form) in self.all_forms.iter(text): if len(form) > max_form_len: best_region_id = region_id max_form_len = len(form) return best_region_id
def get_abridged_normalized_person_name(self): if not self.is_resolved: return normalize_whitespace(self.input_person_name).lower() s = self.family_name.title() if len(self.first_name) >= 1: s += " " + self.first_name[0].upper() + "." if len(self.patronymic) >= 1: s += " " + self.patronymic[0].upper() + "." return s
def normalize_fio_before_db_insert(fio): fio = normalize_whitespace(fio) fio = fio.replace('"', ' ').strip() fio = fio.strip('-') if len(fio) > 0 and fio[0].isdigit(): while len(fio) > 0 and (fio[0].isdigit() or fio[0] == ' ' or fio[0] == '.'): fio = fio[1:] return fio.title()
def get_title_from_smart_parser_json(js): default_value = "null" if js is None: return default_value else: props = js.get('document_sheet_props', []) if len(props) < 1: return default_value return normalize_whitespace(props[0].get('sheet_title', default_value))
def __init__(self, title): self.title = normalize_whitespace(title) self.input_title = title self.type = None self.decl_objects = list() self.decl_time = None self.declarant_positions = list() self.fio = None self.regions = TRussianRegions() self.region_name = None self.org_name = None
def get_region_using_automat(self, automat, text, unknown_region=None): text = normalize_whitespace(text.strip().lower().replace(',', ' ')) for last_pos, (region_id, form) in automat.iter(text): start_pos = last_pos + 1 - len(form) if start_pos > 0: if text[start_pos - 1].isalnum(): continue if last_pos + 1 < len(text): if text[last_pos + 1].isalnum(): continue return region_id, start_pos, last_pos + 1 return unknown_region, None, None
def get_normalized_person_name(self): if self.is_resolved: s = self.family_name.title() + " " + self.first_name.title() if self.first_name_is_abridged: s += '.' if len(self.patronymic) > 0: s += " " + self.patronymic.title() if self.patronymic_is_abridged: s += '.' return s else: return normalize_whitespace(self.input_person_name).lower()
def _build_automatons_without_russia(self): self.nominative_forms = ahocorasick.Automaton() self.all_capitals = ahocorasick.Automaton() for r in self.regions: if r.id == TRussianRegions.Russia_as_s_whole_region_id: continue for f in set([r.name, r.short_name]): self.nominative_forms.add_word(f.lower(), (r.id, f.lower())) capital = normalize_whitespace(r.capital.lower()) self.capitals_to_regions[capital] = r self.all_capitals.add_word(capital, (r.id, capital)) if capital.find('ё') != -1: capital = capital.replace("ё", "е") self.capitals_to_regions[capital] = r self.all_capitals.add_word(capital, (r.id, capital)) self.all_capitals.make_automaton() self.nominative_forms.make_automaton()
def clean_title(title): if title is None: return "" title = normalize_whitespace(title) regexp = "(главная страница)|(Новости)|(Добро пожаловать!)" \ "|(Добро пожаловать)|(None)|(title)|(Title)|(Portal)|(main)|(Срок регистрации домена закончился)" \ "|(403 Forbidden)|(Главная страница сайта)|(Срок действия тарифа истек)" \ "|(Добро пожаловать на сайт)|(Главная страница)|(Main)|(на сайт —)|(Общая информация \|)" title = re.sub(regexp, "", title, re.IGNORECASE).strip(" |:-—") if title.lower() in { "главная", "новости", "основные сведения", "управление", "официальный сайт", "сайт школы", "администрация", "идет проверка..." }: return "" title = re.sub("Главная ([|-])", "", title, re.IGNORECASE).strip(" |:-") return title
def __init__(self, source_file, input_text): self.source_file = source_file self.verdict = DL_RECOGNIZER_ENUM.UNKNOWN if self.source_file is not None: self.smart_parser_person_count, self.smart_parser_title = get_smart_parser_result( self.source_file) else: self.smart_parser_person_count, self.smart_parser_title = 0, "" input_text = normalize_whitespace(input_text) self.start_text = input_text[0:500] self.input_text = input_text self.text_features = defaultdict(TTextFeature) self.description = "" self.normal_russian_text_coef = get_russian_normal_text_ratio( self.input_text) self.find_other_document_types() self.find_header() self.find_other_document_types_in_smart_parser_title()
def all_positions_words(section): if section.position is None: return set() s = normalize_whitespace(section.position).lower() return set(s.split(' '))
def normalize_string(s): return normalize_whitespace(s.lower())