def test_incremental(caplog):
"""Run an end-to-end test on incremental additions."""
caplog.set_level(logging.INFO)
PARALLEL = 1
max_docs = 1
session = Meta.init("postgresql://localhost:5432/" + DB).Session()
docs_path = "tests/data/html/dtc114w.html"
pdf_path = "tests/data/pdf/dtc114w.pdf"
doc_preprocessor = HTMLDocPreprocessor(docs_path, max_docs=max_docs)
corpus_parser = Parser(
session,
parallelism=PARALLEL,
structural=True,
lingual=True,
visual=True,
pdf_path=pdf_path,
)
corpus_parser.apply(doc_preprocessor)
num_docs = session.query(Document).count()
logger.info(f"Docs: {num_docs}")
assert num_docs == max_docs
docs = corpus_parser.get_documents()
last_docs = corpus_parser.get_documents()
assert len(docs[0].sentences) == len(last_docs[0].sentences)
# Mention Extraction
part_ngrams = MentionNgramsPart(parts_by_doc=None, n_max=3)
temp_ngrams = MentionNgramsTemp(n_max=2)
Part = mention_subclass("Part")
Temp = mention_subclass("Temp")
mention_extractor = MentionExtractor(session, [Part, Temp],
[part_ngrams, temp_ngrams],
[part_matcher, temp_matcher])
mention_extractor.apply(docs, parallelism=PARALLEL)
assert session.query(Part).count() == 11
assert session.query(Temp).count() == 8
# Candidate Extraction
PartTemp = candidate_subclass("PartTemp", [Part, Temp])
candidate_extractor = CandidateExtractor(session, [PartTemp],
throttlers=[temp_throttler])
candidate_extractor.apply(docs, split=0, parallelism=PARALLEL)
assert session.query(PartTemp).filter(PartTemp.split == 0).count() == 70
assert session.query(Candidate).count() == 70
# Grab candidate lists
train_cands = candidate_extractor.get_candidates(split=0)
assert len(train_cands) == 1
assert len(train_cands[0]) == 70
# Featurization
featurizer = Featurizer(session, [PartTemp])
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
assert session.query(Feature).count() == 70
assert session.query(FeatureKey).count() == 512
F_train = featurizer.get_feature_matrices(train_cands)
assert F_train[0].shape == (70, 512)
assert len(featurizer.get_keys()) == 512
# Test Dropping FeatureKey
featurizer.drop_keys(["CORE_e1_LENGTH_1"])
assert session.query(FeatureKey).count() == 512
stg_temp_lfs = [
LF_storage_row,
LF_operating_row,
LF_temperature_row,
LF_tstg_row,
LF_to_left,
LF_negative_number_left,
]
labeler = Labeler(session, [PartTemp])
labeler.apply(split=0,
lfs=[stg_temp_lfs],
train=True,
parallelism=PARALLEL)
assert session.query(Label).count() == 70
# Only 5 because LF_operating_row doesn't apply to the first test doc
assert session.query(LabelKey).count() == 5
L_train = labeler.get_label_matrices(train_cands)
assert L_train[0].shape == (70, 5)
assert len(labeler.get_keys()) == 5
docs_path = "tests/data/html/112823.html"
pdf_path = "tests/data/pdf/112823.pdf"
doc_preprocessor = HTMLDocPreprocessor(docs_path, max_docs=max_docs)
corpus_parser.apply(doc_preprocessor, pdf_path=pdf_path, clear=False)
assert len(corpus_parser.get_documents()) == 2
new_docs = corpus_parser.get_last_documents()
assert len(new_docs) == 1
assert new_docs[0].name == "112823"
# Get mentions from just the new docs
mention_extractor.apply(new_docs, parallelism=PARALLEL, clear=False)
assert session.query(Part).count() == 81
assert session.query(Temp).count() == 31
# Just run candidate extraction and assign to split 0
candidate_extractor.apply(new_docs,
split=0,
parallelism=PARALLEL,
clear=False)
# Grab candidate lists
train_cands = candidate_extractor.get_candidates(split=0)
assert len(train_cands) == 1
assert len(train_cands[0]) == 1502
# Update features
featurizer.update(new_docs, parallelism=PARALLEL)
assert session.query(Feature).count() == 1502
assert session.query(FeatureKey).count() == 2573
F_train = featurizer.get_feature_matrices(train_cands)
assert F_train[0].shape == (1502, 2573)
assert len(featurizer.get_keys()) == 2573
# Update Labels
labeler.update(new_docs, lfs=[stg_temp_lfs], parallelism=PARALLEL)
assert session.query(Label).count() == 1502
assert session.query(LabelKey).count() == 6
L_train = labeler.get_label_matrices(train_cands)
assert L_train[0].shape == (1502, 6)
# Test clear
featurizer.clear(train=True)
assert session.query(FeatureKey).count() == 0
def test_incremental():
"""Run an end-to-end test on incremental additions."""
# GitHub Actions gives 2 cores
# help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners
PARALLEL = 2
max_docs = 1
session = Meta.init(CONN_STRING).Session()
docs_path = "tests/data/html/dtc114w.html"
pdf_path = "tests/data/pdf/dtc114w.pdf"
doc_preprocessor = HTMLDocPreprocessor(docs_path, max_docs=max_docs)
corpus_parser = Parser(
session,
parallelism=PARALLEL,
structural=True,
lingual=True,
visual=True,
pdf_path=pdf_path,
)
corpus_parser.apply(doc_preprocessor)
num_docs = session.query(Document).count()
logger.info(f"Docs: {num_docs}")
assert num_docs == max_docs
docs = corpus_parser.get_documents()
last_docs = corpus_parser.get_documents()
assert len(docs[0].sentences) == len(last_docs[0].sentences)
# Mention Extraction
part_ngrams = MentionNgramsPart(parts_by_doc=None, n_max=3)
temp_ngrams = MentionNgramsTemp(n_max=2)
Part = mention_subclass("Part")
Temp = mention_subclass("Temp")
mention_extractor = MentionExtractor(session, [Part, Temp],
[part_ngrams, temp_ngrams],
[part_matcher, temp_matcher])
mention_extractor.apply(docs, parallelism=PARALLEL)
assert session.query(Part).count() == 11
assert session.query(Temp).count() == 8
# Candidate Extraction
PartTemp = candidate_subclass("PartTemp", [Part, Temp])
candidate_extractor = CandidateExtractor(session, [PartTemp],
throttlers=[temp_throttler])
candidate_extractor.apply(docs, split=0, parallelism=PARALLEL)
assert session.query(PartTemp).filter(PartTemp.split == 0).count() == 70
assert session.query(Candidate).count() == 70
# Grab candidate lists
train_cands = candidate_extractor.get_candidates(split=0)
assert len(train_cands) == 1
assert len(train_cands[0]) == 70
# Featurization
featurizer = Featurizer(session, [PartTemp])
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
assert session.query(Feature).count() == 70
assert session.query(FeatureKey).count() == 512
F_train = featurizer.get_feature_matrices(train_cands)
assert F_train[0].shape == (70, 512)
assert len(featurizer.get_keys()) == 512
# Test Dropping FeatureKey
featurizer.drop_keys(["CORE_e1_LENGTH_1"])
assert session.query(FeatureKey).count() == 512
stg_temp_lfs = [
LF_storage_row,
LF_operating_row,
LF_temperature_row,
LF_tstg_row,
LF_to_left,
LF_negative_number_left,
]
labeler = Labeler(session, [PartTemp])
labeler.apply(split=0,
lfs=[stg_temp_lfs],
train=True,
parallelism=PARALLEL)
assert session.query(Label).count() == 70
# Only 5 because LF_operating_row doesn't apply to the first test doc
assert session.query(LabelKey).count() == 5
L_train = labeler.get_label_matrices(train_cands)
assert L_train[0].shape == (70, 5)
assert len(labeler.get_keys()) == 5
docs_path = "tests/data/html/112823.html"
pdf_path = "tests/data/pdf/112823.pdf"
doc_preprocessor = HTMLDocPreprocessor(docs_path, max_docs=max_docs)
corpus_parser.apply(doc_preprocessor, pdf_path=pdf_path, clear=False)
assert len(corpus_parser.get_documents()) == 2
new_docs = corpus_parser.get_last_documents()
assert len(new_docs) == 1
assert new_docs[0].name == "112823"
# Get mentions from just the new docs
mention_extractor.apply(new_docs, parallelism=PARALLEL, clear=False)
assert session.query(Part).count() == 81
assert session.query(Temp).count() == 31
# Test if existing mentions are skipped.
mention_extractor.apply(new_docs, parallelism=PARALLEL, clear=False)
assert session.query(Part).count() == 81
assert session.query(Temp).count() == 31
# Just run candidate extraction and assign to split 0
candidate_extractor.apply(new_docs,
split=0,
parallelism=PARALLEL,
clear=False)
# Grab candidate lists
train_cands = candidate_extractor.get_candidates(split=0)
assert len(train_cands) == 1
assert len(train_cands[0]) == 1502
# Test if existing candidates are skipped.
candidate_extractor.apply(new_docs,
split=0,
parallelism=PARALLEL,
clear=False)
train_cands = candidate_extractor.get_candidates(split=0)
assert len(train_cands) == 1
assert len(train_cands[0]) == 1502
# Update features
featurizer.update(new_docs, parallelism=PARALLEL)
assert session.query(Feature).count() == 1502
assert session.query(FeatureKey).count() == 2573
F_train = featurizer.get_feature_matrices(train_cands)
assert F_train[0].shape == (1502, 2573)
assert len(featurizer.get_keys()) == 2573
# Update LF_storage_row. Now it always returns ABSTAIN.
@labeling_function(name="LF_storage_row")
def LF_storage_row_updated(c):
return ABSTAIN
stg_temp_lfs = [
LF_storage_row_updated,
LF_operating_row,
LF_temperature_row,
LF_tstg_row,
LF_to_left,
LF_negative_number_left,
]
# Update Labels
labeler.update(docs, lfs=[stg_temp_lfs], parallelism=PARALLEL)
labeler.update(new_docs, lfs=[stg_temp_lfs], parallelism=PARALLEL)
assert session.query(Label).count() == 1502
# Only 5 because LF_storage_row doesn't apply to any doc (always ABSTAIN)
assert session.query(LabelKey).count() == 5
L_train = labeler.get_label_matrices(train_cands)
assert L_train[0].shape == (1502, 5)
# Test clear
featurizer.clear(train=True)
assert session.query(FeatureKey).count() == 0
def test_feature_extraction():
"""Test extracting candidates from mentions from documents."""
PARALLEL = 1
max_docs = 1
session = Meta.init(CONN_STRING).Session()
docs_path = "tests/data/html/"
pdf_path = "tests/data/pdf/"
# Parsing
logger.info("Parsing...")
doc_preprocessor = HTMLDocPreprocessor(docs_path, max_docs=max_docs)
corpus_parser = Parser(session,
structural=True,
lingual=True,
visual=True,
pdf_path=pdf_path)
corpus_parser.apply(doc_preprocessor, parallelism=PARALLEL)
assert session.query(Document).count() == max_docs
assert session.query(Sentence).count() == 799
docs = session.query(Document).order_by(Document.name).all()
# Mention Extraction
part_ngrams = MentionNgrams(n_max=1)
temp_ngrams = MentionNgrams(n_max=1)
Part = mention_subclass("Part")
Temp = mention_subclass("Temp")
mention_extractor = MentionExtractor(session, [Part, Temp],
[part_ngrams, temp_ngrams],
[part_matcher, temp_matcher])
mention_extractor.apply(docs, parallelism=PARALLEL)
assert docs[0].name == "112823"
assert session.query(Part).count() == 58
assert session.query(Temp).count() == 16
part = session.query(Part).order_by(Part.id).all()[0]
temp = session.query(Temp).order_by(Temp.id).all()[0]
logger.info(f"Part: {part.context}")
logger.info(f"Temp: {temp.context}")
# Candidate Extraction
PartTemp = candidate_subclass("PartTemp", [Part, Temp])
candidate_extractor = CandidateExtractor(session, [PartTemp])
candidate_extractor.apply(docs, split=0, parallelism=PARALLEL)
n_cands = session.query(PartTemp).count()
# Featurization based on default feature library
featurizer = Featurizer(session, [PartTemp])
# Test that featurization default feature library
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_default_feats = session.query(FeatureKey).count()
featurizer.clear(train=True)
# Example feature extractor
def feat_ext(candidates):
candidates = candidates if isinstance(candidates,
list) else [candidates]
for candidate in candidates:
yield candidate.id, f"cand_id_{candidate.id}", 1
# Featurization with one extra feature extractor
feature_extractors = FeatureExtractor(customize_feature_funcs=[feat_ext])
featurizer = Featurizer(session, [PartTemp],
feature_extractors=feature_extractors)
# Test that featurization default feature library with one extra feature extractor
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_default_w_customized_features = session.query(FeatureKey).count()
featurizer.clear(train=True)
# Featurization with only textual feature
feature_extractors = FeatureExtractor(features=["textual"])
featurizer = Featurizer(session, [PartTemp],
feature_extractors=feature_extractors)
# Test that featurization textual feature library
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_textual_features = session.query(FeatureKey).count()
featurizer.clear(train=True)
# Featurization with only tabular feature
feature_extractors = FeatureExtractor(features=["tabular"])
featurizer = Featurizer(session, [PartTemp],
feature_extractors=feature_extractors)
# Test that featurization tabular feature library
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_tabular_features = session.query(FeatureKey).count()
featurizer.clear(train=True)
# Featurization with only structural feature
feature_extractors = FeatureExtractor(features=["structural"])
featurizer = Featurizer(session, [PartTemp],
feature_extractors=feature_extractors)
# Test that featurization structural feature library
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_structural_features = session.query(FeatureKey).count()
featurizer.clear(train=True)
# Featurization with only visual feature
feature_extractors = FeatureExtractor(features=["visual"])
featurizer = Featurizer(session, [PartTemp],
feature_extractors=feature_extractors)
# Test that featurization visual feature library
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_visual_features = session.query(FeatureKey).count()
featurizer.clear(train=True)
assert (n_default_feats == n_textual_features + n_tabular_features +
n_structural_features + n_visual_features)
assert n_default_w_customized_features == n_default_feats + n_cands
def test_unary_relation_feature_extraction():
"""Test extracting unary candidates from mentions from documents."""
PARALLEL = 1
max_docs = 1
session = Meta.init(CONN_STRING).Session()
docs_path = "tests/data/html/"
pdf_path = "tests/data/pdf/"
# Parsing
logger.info("Parsing...")
doc_preprocessor = HTMLDocPreprocessor(docs_path, max_docs=max_docs)
corpus_parser = Parser(session,
structural=True,
lingual=True,
visual=True,
pdf_path=pdf_path)
corpus_parser.apply(doc_preprocessor, parallelism=PARALLEL)
assert session.query(Document).count() == max_docs
assert session.query(Sentence).count() == 799
docs = session.query(Document).order_by(Document.name).all()
# Mention Extraction
part_ngrams = MentionNgrams(n_max=1)
Part = mention_subclass("Part")
mention_extractor = MentionExtractor(session, [Part], [part_ngrams],
[part_matcher])
mention_extractor.apply(docs, parallelism=PARALLEL)
assert docs[0].name == "112823"
assert session.query(Part).count() == 58
part = session.query(Part).order_by(Part.id).all()[0]
logger.info(f"Part: {part.context}")
# Candidate Extraction
PartRel = candidate_subclass("PartRel", [Part])
candidate_extractor = CandidateExtractor(session, [PartRel])
candidate_extractor.apply(docs, split=0, parallelism=PARALLEL)
# Featurization based on default feature library
featurizer = Featurizer(session, [PartRel])
# Test that featurization default feature library
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_default_feats = session.query(FeatureKey).count()
featurizer.clear(train=True)
# Featurization with only textual feature
feature_extractors = FeatureExtractor(features=["textual"])
featurizer = Featurizer(session, [PartRel],
feature_extractors=feature_extractors)
# Test that featurization textual feature library
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_textual_features = session.query(FeatureKey).count()
featurizer.clear(train=True)
# Featurization with only tabular feature
feature_extractors = FeatureExtractor(features=["tabular"])
featurizer = Featurizer(session, [PartRel],
feature_extractors=feature_extractors)
# Test that featurization tabular feature library
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_tabular_features = session.query(FeatureKey).count()
featurizer.clear(train=True)
# Featurization with only structural feature
feature_extractors = FeatureExtractor(features=["structural"])
featurizer = Featurizer(session, [PartRel],
feature_extractors=feature_extractors)
# Test that featurization structural feature library
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_structural_features = session.query(FeatureKey).count()
featurizer.clear(train=True)
# Featurization with only visual feature
feature_extractors = FeatureExtractor(features=["visual"])
featurizer = Featurizer(session, [PartRel],
feature_extractors=feature_extractors)
# Test that featurization visual feature library
featurizer.apply(split=0, train=True, parallelism=PARALLEL)
n_visual_features = session.query(FeatureKey).count()
featurizer.clear(train=True)
assert (n_default_feats == n_textual_features + n_tabular_features +
n_structural_features + n_visual_features)