def get_subclasses(experiment): # 1.) Mention subclasses Data = mention_subclass("Data") Row = mention_subclass("Row") Col = mention_subclass("Col") # 2.) Mention spaces data_ngrams = MentionSentences() # MentionNgrams(n_max=3) row_ngrams = MentionSentences() # MentionNgrams(n_min=1, n_max=8) col_ngrams = MentionSentences() # MentionNgrams(n_min=1, n_max=8) # 3.) Matchers data_regex_matcher = RegexMatchSpan(rgx=r"[0-9-,.%$#]+( to | )?[0-9-,.%$#]*|^x$", longest_match_only=True) data_label_matcher = LambdaFunctionMatcher(func=get_label_matcher("Data", experiment)) data_matcher = Intersect(data_regex_matcher, data_label_matcher) row_regex_matcher = RegexMatchSpan(rgx=r"^.*$", longest_match_only=True) row_label_matcher = LambdaFunctionMatcher(func=get_label_matcher("Header", experiment)) row_matcher = Intersect(row_regex_matcher, row_label_matcher) col_regex_matcher = RegexMatchSpan(rgx=r"^.*$", longest_match_only=True) col_label_matcher = LambdaFunctionMatcher(func=get_label_matcher("Header", experiment)) col_matcher = Intersect(col_regex_matcher, col_label_matcher) # 4.) Candidate classes RowCandidate = candidate_subclass("RowCandidate", [Data, Row]) ColCandidate = candidate_subclass("ColCandidate", [Data, Col]) # 5.) Throttlers mention_classes = [Data, Row, Col] mention_spaces = [data_ngrams, row_ngrams, col_ngrams] matchers = [data_matcher, row_matcher, col_matcher] candidate_classes = [RowCandidate, ColCandidate] throttlers = [row_filter, col_filter] return (mention_classes, mention_spaces, matchers, candidate_classes, throttlers)
def get_supply_current_matcher(): def current_units(attr): # NOTE: These two symbols for mu are unique, not duplicates. current_units = ["ma", "μa", "ua", "µa", "\uf06da"] keywords = ["supply", "quiescent", "iq", "is", "idd", "icc"] filter_keywords = ["offset", "bias", "logic", "shutdown"] related_ngrams = set(get_right_ngrams(attr, n_max=1, lower=True)) related_ngrams.update(get_row_ngrams(attr, n_max=1, spread=[-5, 5], lower=True)) if attr.get_span().strip() == "0": return False if overlap(filter_keywords, get_row_ngrams(attr, n_max=1, lower=True)): return False if overlap(current_units, related_ngrams) and overlap(keywords, related_ngrams): return True return False # match 4-digit integers, or two-digit floats up with 2 points of precision current_rgx = RegexMatchSpan( rgx=r"(±?\d{1,2}\.\d{1,2}|±?\d{1,4})", longest_match_only=False ) current_lambda = LambdaFunctionMatcher(func=current_units) condition_lambda = LambdaFunctionMatcher(func=_condition) location_lambda = LambdaFunctionMatcher(func=_first_page_or_table) return Intersect(condition_lambda, location_lambda, current_rgx, current_lambda)
def phone_extract(docs, session, phone_subclass, parallelism, clear=True): phone_lambda_matcher = LambdaFunctionMatcher(func=matcher_number_phone) regex_matcher = LambdaFunctionMatcher(func=regexMatch) phone_lamda_matcher = Union(regex_matcher, phone_lambda_matcher) phone_space = MentionPhoneNumber() mention_extractor = MentionExtractor(session, [phone_subclass], [phone_space], [phone_lamda_matcher]) mention_extractor.apply(docs, parallelism=parallelism, clear=clear)
def phone_extract_server(document, phone_subclass): phone_lambda_matcher = LambdaFunctionMatcher(func=matcher_number_phone) regex_matcher = LambdaFunctionMatcher(func=regexMatch) phone_lamda_matcher = Union(regex_matcher, phone_lambda_matcher) phone_space = MentionPhoneNumber() document = MentionExtractorUDF([phone_subclass], [phone_space], [phone_lamda_matcher]).apply(document) return document
def test_mention_longest_match(): """Test longest match filtering in mention extraction.""" file_name = "lincoln_short" docs_path = f"tests/data/pure_html/{file_name}.html" doc = parse_doc(docs_path, file_name) # Mention Extraction name_ngrams = MentionNgramsPart(n_max=3) place_ngrams = MentionNgramsTemp(n_max=4) Name = mention_subclass("Name") Place = mention_subclass("Place") def is_birthplace_table_row(mention): if not mention.sentence.is_tabular(): return False ngrams = get_row_ngrams(mention, lower=True) if "birth_place" in ngrams: return True else: return False birthplace_matcher = LambdaFunctionMatcher( func=is_birthplace_table_row, longest_match_only=False ) mention_extractor_udf = MentionExtractorUDF( [Name, Place], [name_ngrams, place_ngrams], [PersonMatcher(), birthplace_matcher], ) doc = mention_extractor_udf.apply(doc) mentions = doc.places mention_spans = [x.context.get_span() for x in mentions] assert "Sinking Spring Farm" in mention_spans assert "Farm" in mention_spans assert len(mention_spans) == 23 # Clear manually for mention in doc.places[:]: doc.places.remove(mention) birthplace_matcher = LambdaFunctionMatcher( func=is_birthplace_table_row, longest_match_only=True ) mention_extractor_udf = MentionExtractorUDF( [Name, Place], [name_ngrams, place_ngrams], [PersonMatcher(), birthplace_matcher], ) doc = mention_extractor_udf.apply(doc) mentions = doc.places mention_spans = [x.context.get_span() for x in mentions] assert "Sinking Spring Farm" in mention_spans assert "Farm" not in mention_spans assert len(mention_spans) == 4
def address_extract_server(document, address_subclass): address_m1 = LambdaFunctionMatcher(func = has_province_address) address_m2 = LambdaFunctionMatcher(func = has_geographic_term_address) address_m3 = LambdaFunctionMatcher(func = address_prefix) address_m4 = LambdaFunctionMatcher(func = is_collection_of_number_and_geographical_term_and_provinces_name_address) address_m5 = LambdaFunctionMatcher(func = hasnt_ignor_words) address_matcher = Intersect(Union(address_m1, address_m2, address_m3), address_m4, address_m5) address_space = MentionSentences() document = MentionExtractorUDF([address_subclass], [address_space], [address_matcher]).apply(document) return document
def address_extract(docs, session, address_subclass, parallelism, clear=True): address_m1 = LambdaFunctionMatcher(func = has_province_address) address_m2 = LambdaFunctionMatcher(func = has_geographic_term_address) address_m3 = LambdaFunctionMatcher(func = address_prefix) address_m4 = LambdaFunctionMatcher(func = is_collection_of_number_and_geographical_term_and_provinces_name_address) address_m5 = LambdaFunctionMatcher(func = hasnt_ignor_words) address_matcher = Intersect(Union(address_m1, address_m2, address_m3), address_m4, address_m5) address_space = MentionSentences() mention_extractor = MentionExtractor(session, [address_subclass], [address_space], [address_matcher]) mention_extractor.apply(docs, parallelism=parallelism,clear=clear)
def birthday_extract_server(document, birthday_subclass): filter_birthday_matcher = LambdaFunctionMatcher(func=filter_birthday, longest_match_only=True) birthday_conditions_matcher = LambdaFunctionMatcher( func=birthday_conditions, longest_match_only=True) birthday_matcher = Intersect(filter_birthday_matcher, birthday_conditions_matcher) birthday_space = MentionDates() document = MentionExtractorUDF([birthday_subclass], [birthday_space], [birthday_matcher]).apply(document) return document
def _get_part_matcher(): """Return the part matcher.""" # Transistor Naming Conventions as Regular Expressions eeca_rgx = (r"([ABC][A-Z][WXYZ]?[0-9]{3,5}(?:[A-Z]){0,5}[0-9]?[A-Z]?" r"(?:-[A-Z0-9]{1,7})?(?:[-][A-Z0-9]{1,2})?(?:\/DG)?)") jedec_rgx = r"(2N\d{3,4}[A-Z]{0,5}[0-9]?[A-Z]?)" jis_rgx = r"(2S[ABCDEFGHJKMQRSTVZ]{1}[\d]{2,4})" others_rgx = ( r"((?:NSVBC|SMBT|MJ|MJE|MPS|MRF|RCA|TIP|ZTX|ZT|ZXT|TIS|TIPL|DTC|MMBT" r"|SMMBT|PZT|FZT|STD|BUV|PBSS|KSC|CXT|FCX|CMPT){1}[\d]{2,4}[A-Z]{0,5}" r"(?:-[A-Z0-9]{0,6})?(?:[-][A-Z0-9]{0,1})?)") part_rgx = "|".join([eeca_rgx, jedec_rgx, jis_rgx, others_rgx]) add_rgx = r"^[A-Z0-9\-]{5,15}$" part_file_name_lambda_matcher = LambdaFunctionMatcher( func=_part_file_name_conditions) part_rgx_matcher = RegexMatchSpan(rgx=part_rgx, longest_match_only=True) part_dict_matcher = DictionaryMatch(d=_get_digikey_parts_set(DICT_PATH)) part_file_name_matcher = Intersect( RegexMatchSpan(rgx=add_rgx, longest_match_only=True), part_file_name_lambda_matcher, ) return Union(part_rgx_matcher, part_dict_matcher, part_file_name_matcher)
def birthday_extract(docs, session, birthday_subclass, parallelism, clear=True): filter_birthday_matcher = LambdaFunctionMatcher(func=filter_birthday, longest_match_only=True) birthday_conditions_matcher = LambdaFunctionMatcher( func=birthday_conditions, longest_match_only=True) birthday_matcher = Intersect(filter_birthday_matcher, birthday_conditions_matcher) birthday_space = MentionDates() mention_extractor = MentionExtractor(session, [birthday_subclass], [birthday_space], [birthday_matcher]) mention_extractor.apply(docs, parallelism=parallelism, clear=clear)
def email_extract(docs, session, email_subclass, parallelism, clear=True): email_matcher = LambdaFunctionMatcher(func=email_mc, longest_match_only=True) email_space = MentionEmails() mention_extractor = MentionExtractor(session, [email_subclass], [email_space], [email_matcher]) mention_extractor.apply(docs, parallelism=parallelism, clear=clear)
def test_row_col_ngram_extraction(): """Test whether row/column ngrams list is empty, if mention is not in a table.""" file_name = "lincoln_short" docs_path = f"tests/data/pure_html/{file_name}.html" doc = parse_doc(docs_path, file_name) # Mention Extraction place_ngrams = MentionNgramsTemp(n_max=4) Place = mention_subclass("Place") def get_row_and_column_ngrams(mention): row_ngrams = list(get_row_ngrams(mention)) col_ngrams = list(get_col_ngrams(mention)) if not mention.sentence.is_tabular(): assert len(row_ngrams) == 1 and row_ngrams[0] is None assert len(col_ngrams) == 1 and col_ngrams[0] is None else: assert not any(x is None for x in row_ngrams) assert not any(x is None for x in col_ngrams) if "birth_place" in row_ngrams: return True else: return False birthplace_matcher = LambdaFunctionMatcher(func=get_row_and_column_ngrams) mention_extractor_udf = MentionExtractorUDF( [Place], [place_ngrams], [birthplace_matcher] ) doc = mention_extractor_udf.apply(doc)
def email_extract_server(document, email_subclass): email_matcher = LambdaFunctionMatcher(func=email_mc, longest_match_only=True) email_space = MentionEmails() document = MentionExtractorUDF([email_subclass], [email_space], [email_matcher]).apply(document) return document
def get_gain_matcher(): def hertz_units(attr): hertz_units = ["mhz", "khz"] keywords = [ "product", "gain", "gain", "unity", "bandwidth", "gbp", "gbw", "gbwp", ] filter_keywords = ["-3 db", "maximum", "minimum", "impedance"] related_ngrams = set(get_right_ngrams(attr, n_max=1, lower=True)) related_ngrams.update( get_row_ngrams(attr, n_max=1, spread=[-2, 2], lower=True)) cell_ngrams = set(get_cell_ngrams(attr, n_max=1, lower=True)) if "f" in cell_ngrams and "=" in cell_ngrams: return False if attr.get_span().strip() == "0": return False if overlap(filter_keywords, get_row_ngrams(attr, n_max=1, lower=True)): return False if overlap(hertz_units, related_ngrams) and overlap( keywords, related_ngrams): return True return False # match 3-digit integers, or two-digit floats up with 2 points of precision gain_rgx = RegexMatchSpan(rgx=r"^(?:\d{1,2}\.\d{1,2}|\d{1,3})$", longest_match_only=False) hertz_lambda = LambdaFunctionMatcher(func=hertz_units) condition_lambda = LambdaFunctionMatcher(func=_condition) location_lambda = LambdaFunctionMatcher(func=_first_page_or_table) return Intersect(hertz_lambda, gain_rgx, location_lambda, condition_lambda)
def test_lambda_function_matcher(doc_setup): """Test DictionaryMatch matcher.""" doc = doc_setup space = MentionNgrams(n_min=1, n_max=1) # Test with a lambda function matcher = LambdaFunctionMatcher(func=lambda x: True) assert set(tc.get_span() for tc in matcher.apply(space.apply(doc))) == { "This", "is", "apple", } # Test if matcher raises an error when _f is given non-TemporarySpanMention with pytest.raises(ValueError): list(matcher.apply(doc.sentences[0].words)) # Test if an error raised when a func is not provided. with pytest.raises(Exception): LambdaFunctionMatcher()
def _get_polarity_matcher(): """Return the polarity matcher.""" def polarity_conditions(attr): return not overlap(["complement", "complementary"], get_sentence_ngrams(attr)) polarity_rgx_matcher = RegexMatchSpan(rgx=r"NPN|PNP", longest_match_only=False, ignore_case=True) polarity_lambda_matcher = LambdaFunctionMatcher(func=polarity_conditions) return Intersect(polarity_rgx_matcher, polarity_lambda_matcher)
def _get_ce_v_max_matcher(): """Return a collector-emmiter voltage max matcher.""" ce_keywords = set( ["collector emitter", "collector-emitter", "collector - emitter"]) ce_abbrevs = set(["ceo", "vceo"]) def ce_v_max_conditions(attr): ngrams = set(get_row_ngrams(attr, n_max=1)) if not overlap(ce_keywords.union(ce_abbrevs), ngrams): return False if any(_ in attr.sentence.text.lower() for _ in ["vcb", "base"]): return False return True ce_v_max_rgx_matcher = RegexMatchSpan(rgx=r"\b\d{1,2}[05]", longest_match_only=False) ce_v_max_row_matcher = LambdaFunctionMatcher(func=ce_v_max_conditions) ce_v_max_in_table = LambdaFunctionMatcher(func=_attr_in_table) return Intersect(ce_v_max_rgx_matcher, ce_v_max_row_matcher, ce_v_max_in_table)
def test_row_col_ngram_extraction(caplog): """Test whether row/column ngrams list is empty, if mention is not in a table.""" caplog.set_level(logging.INFO) PARALLEL = 1 max_docs = 1 session = Meta.init("postgresql://localhost:5432/" + DB).Session() docs_path = "tests/data/pure_html/lincoln_short.html" # Parsing logger.info("Parsing...") doc_preprocessor = HTMLDocPreprocessor(docs_path, max_docs=max_docs) corpus_parser = Parser(session, structural=True, lingual=True) corpus_parser.apply(doc_preprocessor, parallelism=PARALLEL) docs = session.query(Document).order_by(Document.name).all() # Mention Extraction place_ngrams = MentionNgramsTemp(n_max=4) Place = mention_subclass("Place") def get_row_and_column_ngrams(mention): row_ngrams = list(get_row_ngrams(mention)) col_ngrams = list(get_col_ngrams(mention)) if not mention.sentence.is_tabular(): assert len(row_ngrams) == 1 and row_ngrams[0] is None assert len(col_ngrams) == 1 and col_ngrams[0] is None else: assert not any(x is None for x in row_ngrams) assert not any(x is None for x in col_ngrams) if "birth_place" in row_ngrams: return True else: return False birthplace_matcher = LambdaFunctionMatcher(func=get_row_and_column_ngrams) mention_extractor = MentionExtractor( session, [Place], [place_ngrams], [birthplace_matcher] ) mention_extractor.apply(docs, parallelism=PARALLEL)
def name_extract(docs, session, name_subclass, parallelism, clear=True): length_name_matcher = LambdaFunctionMatcher(func=length_name) position_name_matcher = LambdaFunctionMatcher(func=position_name) capitalize_name_matcher = LambdaFunctionMatcher(func=capitalize_name) last_name_matcher = LambdaFunctionMatcher(func=last_name) name_common_matcher = LambdaFunctionMatcher(func=name_common) check_name_matcher = LambdaFunctionMatcher(func=check_name) prefix_name_matcher = LambdaFunctionMatcher(func=prefix_name) form_name_matcher = Intersect(length_name_matcher, position_name_matcher, capitalize_name_matcher) name_matcher = Intersect( Union(Intersect(last_name_matcher, form_name_matcher), Intersect(name_common_matcher, form_name_matcher), prefix_name_matcher), check_name_matcher) name_space = MentionName() mention_extractor = MentionExtractor(session, [name_subclass], [name_space], [name_matcher]) mention_extractor.apply(docs, parallelism=parallelism, clear=clear)
def name_extract_server(document, name_subclass): length_name_matcher = LambdaFunctionMatcher(func=length_name) position_name_matcher = LambdaFunctionMatcher(func=position_name) capitalize_name_matcher = LambdaFunctionMatcher(func=capitalize_name) last_name_matcher = LambdaFunctionMatcher(func=last_name) name_common_matcher = LambdaFunctionMatcher(func=name_common) check_name_matcher = LambdaFunctionMatcher(func=check_name) prefix_name_matcher = LambdaFunctionMatcher(func=prefix_name) form_name_matcher = Intersect(length_name_matcher, position_name_matcher, capitalize_name_matcher) name_matcher = Intersect( Union(Intersect(last_name_matcher, form_name_matcher), Intersect(name_common_matcher, form_name_matcher), prefix_name_matcher), check_name_matcher) name_space = MentionName() document = MentionExtractorUDF([name_subclass], [name_space], [name_matcher]).apply(document) return document
Presidentname = mention_subclass("Presidentname") Placeofbirth = mention_subclass("Placeofbirth") def mention_span_matches_file_name(mention): president_name_string = mention.get_span() file_name = mention.sentence.document.name.replace("_", " ") if president_name_string == file_name: return True else: return False from fonduer.candidates.matchers import LambdaFunctionMatcher, Intersect, Union president_name_matcher = LambdaFunctionMatcher( func=mention_span_matches_file_name) from fonduer.utils.data_model_utils import get_row_ngrams def is_in_birthplace_table_row(mention): if not mention.sentence.is_tabular(): return False ngrams = get_row_ngrams(mention, lower=True) birth_place_words = set(["birth", "place"]) if birth_place_words <= set(ngrams): return True else: return False
# Getting all documents parsed by Snorkel print("Getting documents and sentences...") docs = session.query(Document).all() #sents = session.query(Sentence).all() from fonduer.candidates import CandidateExtractor, MentionExtractor, MentionNgrams from fonduer.candidates.models import mention_subclass, candidate_subclass from fonduer.candidates.matchers import RegexMatchSpan, Union, LambdaFunctionMatcher from dataset_utils import price_match # Defining ngrams for candidates extraction_name = 'price' ngrams = MentionNgrams(n_max=5) # Define matchers matchers = LambdaFunctionMatcher(func=price_match) # Getting candidates PriceMention = mention_subclass("PriceMention") mention_extractor = MentionExtractor( session, [PriceMention], [ngrams], [matchers] ) mention_extractor.clear_all() mention_extractor.apply(docs, parallelism=parallelism) candidate_class = candidate_subclass("Price", [PriceMention]) candidate_extractor = CandidateExtractor(session, [candidate_class]) # Applying candidate extractors candidate_extractor.apply(docs, split=0, parallelism=parallelism) print("==============================") print(f"Candidate extraction results for {postgres_db_name}:")
from fonduer.candidates.matchers import LambdaFunctionMatcher def person_name_matcher(mention): mention_set = set(mention.sentence.ner_tags) if len(mention_set) == 1 and 'PERSON' in mention_set: return True else: return False person_name_function = LambdaFunctionMatcher(func=person_name_matcher) def get_matchers(): return [person_name_function]
def test_mention_longest_match(caplog): """Test longest match filtering in mention extraction.""" caplog.set_level(logging.INFO) # SpaCy on mac has issue on parallel parsing PARALLEL = 1 max_docs = 1 session = Meta.init("postgresql://localhost:5432/" + DB).Session() docs_path = "tests/data/pure_html/lincoln_short.html" # Parsing logger.info("Parsing...") doc_preprocessor = HTMLDocPreprocessor(docs_path, max_docs=max_docs) corpus_parser = Parser(session, structural=True, lingual=True) corpus_parser.apply(doc_preprocessor, parallelism=PARALLEL) docs = session.query(Document).order_by(Document.name).all() # Mention Extraction name_ngrams = MentionNgramsPart(n_max=3) place_ngrams = MentionNgramsTemp(n_max=4) Name = mention_subclass("Name") Place = mention_subclass("Place") def is_birthplace_table_row(mention): if not mention.sentence.is_tabular(): return False ngrams = get_row_ngrams(mention, lower=True) if "birth_place" in ngrams: return True else: return False birthplace_matcher = LambdaFunctionMatcher( func=is_birthplace_table_row, longest_match_only=False ) mention_extractor = MentionExtractor( session, [Name, Place], [name_ngrams, place_ngrams], [PersonMatcher(), birthplace_matcher], ) mention_extractor.apply(docs, parallelism=PARALLEL) mentions = session.query(Place).all() mention_spans = [x.context.get_span() for x in mentions] assert "Sinking Spring Farm" in mention_spans assert "Farm" in mention_spans assert len(mention_spans) == 23 birthplace_matcher = LambdaFunctionMatcher( func=is_birthplace_table_row, longest_match_only=True ) mention_extractor = MentionExtractor( session, [Name, Place], [name_ngrams, place_ngrams], [PersonMatcher(), birthplace_matcher], ) mention_extractor.apply(docs, parallelism=PARALLEL) mentions = session.query(Place).all() mention_spans = [x.context.get_span() for x in mentions] assert "Sinking Spring Farm" in mention_spans assert "Farm" not in mention_spans assert len(mention_spans) == 4
from fonduer.candidates import CandidateExtractor, MentionExtractor, MentionNgrams from fonduer.candidates.models import mention_subclass, candidate_subclass from dataset_utils import price_match_hour, price_match_half from fonduer.candidates.matchers import LambdaFunctionMatcher # Defining ngrams for candidates if args['name'] == 'hour': extraction_name = 'price_per_hour' elif args['name'] == 'half_hour': extraction_name = 'price_per_half_hour' ngrams = MentionNgrams(n_max=1) # Define matchers if args['name'] == 'hour': price_matcher = LambdaFunctionMatcher(func=price_match_hour) elif args['name'] == 'half_hour': price_matcher = LambdaFunctionMatcher(func=price_match_half) matchers = price_matcher # Getting candidates PriceMention = mention_subclass("PriceMention") mention_extractor = MentionExtractor(session, [PriceMention], [ngrams], [matchers]) mention_extractor.clear_all() mention_extractor.apply(docs, parallelism=parallelism) candidate_class = candidate_subclass("Price", [PriceMention]) candidate_extractor = CandidateExtractor(session, [candidate_class]) # Applying candidate extractors
def part_file_name_conditions(attr): """Check part file name conditions.""" file_name = attr.sentence.document.name if len(file_name.split("_")) != 2: return False if attr.get_span()[0] == "-": return False name = attr.get_span().replace("-", "") return (any(char.isdigit() for char in name) and any(char.isalpha() for char in name) and common_prefix_length_diff(file_name.split("_")[1], name) <= 2) add_rgx = r"^[A-Z0-9\-]{5,15}$" part_file_name_lambda_matcher = LambdaFunctionMatcher( func=part_file_name_conditions) part_file_name_matcher = Intersect( RegexMatchSpan(rgx=add_rgx, longest_match_only=True), part_file_name_lambda_matcher) part_matcher = Union(part_rgx_matcher, part_dict_matcher, part_file_name_matcher) # CE Voltage Matcher ce_keywords = set( ["collector emitter", "collector-emitter", "collector - emitter"]) ce_abbrevs = set(["ceo", "vceo"]) ce_v_max_rgx_matcher = RegexMatchSpan(rgx=r"\d{1,2}[05]", longest_match_only=False)
#sents = session.query(Sentence).all() from fonduer.candidates import CandidateExtractor, MentionExtractor, MentionNgrams from fonduer.candidates.models import mention_subclass, candidate_subclass from dataset_utils import LocationMatcher, city_index from fonduer.candidates.matchers import Union, LambdaFunctionMatcher, Intersect from emmental_utils import get_posting_html_fast # Defining ngrams for candidates extraction_name = 'location' ngrams = MentionNgrams(n_max=3) # Define matchers # Geolocation matcher cities = city_index('../utils/data/cities15000.txt') geo_location_matcher = LambdaFunctionMatcher(func=cities.fast_loc) # In raw text matcher with open(f"{config['prediction_model_path']}/char_dict.pkl", 'rb') as fl: char_dict = pickle.load(fl) dataset = load_data_from_db(postgres_db_name, config['postgres_location'], {}, char_dict=char_dict, clobber_label=True) text_dict = {a[0]['uid']: a[0]['text'] for a in dataset} def post_matcher_fun(m): term = r"([Ll]ocation:[\w\W]{1,200}</.{0,20}>|\W[cC]ity:[\w\W]{1,200}</.{0,20}>|\d\dyo\W|\d\d.{0,10}\Wyo\W|\d\d.{0,10}\Wold\W|\d\d.{0,10}\Wyoung\W|\Wage\W.{0,10}\d\d)" #if m.get_span() in get_posting_html_fast(m.sentence.document.text, term): if m.get_span() in text_dict[m.sentence.document.name]: