def test_sum():
my_list = Datasource("my_list")
my_sum = aggregators.sum(my_list)
cache = {my_list: [1, 2, 3, 4]}
assert solve(my_sum, cache=cache) == 10
cache = {my_list: []}
assert solve(my_sum, cache=cache) == 0
cache = {my_list: None}
assert solve(my_sum, cache=cache) == 0
assert str(my_sum) == "feature.sum(<datasource.my_list>)"
assert pickle.loads(pickle.dumps(my_sum)) == my_sum
def test_sum_vectors():
my_list = Datasource("my_list")
my_sum = aggregators.sum(my_list, vector=True)
cache = {my_list: [[1, 2, 3], [4, 5, 6]]}
assert all(a == b for a, b in zip(solve(my_sum, cache=cache), [5, 7, 9]))
cache = {my_list: [[]]}
assert solve(my_sum, cache=cache) == [0]
cache = {my_list: [None]}
assert solve(my_sum, cache=cache) == [0]
assert str(my_sum) == "feature_vector.sum(<datasource.my_list>)"
assert pickle.loads(pickle.dumps(my_sum)) == my_sum
r"File|Image\:", name="enwiki.revision.image_links")
# References
revision = Revision(
"enwiki.revision.revision",
wikitext.revision.datasources,
)
paragraphs = mappers.map(str,
revision.paragraphs_sentences_and_whitespace,
name="enwiki.revision.paragraphs")
paragraphs_without_refs = filters.regex_matching(
r"^(?!\s*$)((?!<ref>)(.|\n))*$",
paragraphs,
name="enwiki.revision.paragraphs_without_refs")
paragraphs_without_refs_total_length = aggregators.sum(
mappers.map(len, paragraphs_without_refs),
name="enwiki.revision.paragraphs_without_refs_total_length")
local_wiki = [
image_links,
image_links / max(wikitext.revision.content_chars, 1),
category_links,
category_links / max(wikitext.revision.content_chars, 1),
cite_templates,
cite_templates / max(wikitext.revision.content_chars, 1),
proportion_of_templated_references,
non_templated_references,
non_templated_references / max(wikitext.revision.content_chars, 1),
non_cite_templates,
non_cite_templates / max(wikitext.revision.content_chars, 1),
infobox_templates,
revision_lang_map = Datasource(
"revision.lang_map", process_normalized_lang_map,
depends_on=[ro.revision.text])
parent_lang_map = Datasource(
"revision.parent.lang_map", process_normalized_lang_map,
depends_on=[ro.revision.parent.text])
parent_lang_vector = vectorizers.vectorize(
parent_lang_map, keys=ALL_NORMALIZED_LANGS, returns=float,
name="revision.parent.lang_vector")
lang_delta = frequencies.delta(parent_lang_map, revision_lang_map)
lang_delta_vector = vectorizers.vectorize(
lang_delta, keys=ALL_NORMALIZED_LANGS, returns=float,
name="revision.diff.lang_delta_vector")
lang_delta_sum_diff = aggregators.sum(
mappers.abs(dicts.values(lang_delta)),
name="revision.diff.lang_delta_sum_diff")
def process_title_lang_match(title_lang, lang_delta):
return lang_delta.get(title_lang, 0.0)
parent_lang_match = Feature("revision.parent.lang_match",
process_title_lang_match,
depends_on=[translation_title_lang,
parent_lang_map],
returns=float)
match_lang_delta = Feature("revision.diff.match_lang_delta",
process_title_lang_match,
depends_on=[translation_title_lang,
# References
revision = Revision(
"glwiki.revision.revision",
wikitext.revision.datasources,
)
paragraphs = mappers.map(
str, revision.paragraphs_sentences_and_whitespace,
name="glwiki.revision.paragraphs"
)
paragraphs_without_refs = filters.regex_matching(
r"^(?!\s*$)((?!<ref>)(.|\n))*$",
paragraphs,
name="glwiki.revision.paragraphs_without_refs"
)
paragraphs_without_refs_total_length = aggregators.sum(
mappers.map(len, paragraphs_without_refs),
name="glwiki.revision.paragraphs_without_refs_total_length"
)
local_wiki = [
image_links,
image_links / max(wikitext.revision.content_chars, 1),
category_links,
category_links / max(wikitext.revision.content_chars, 1),
cn_templates + 1,
cn_templates / max(wikitext.revision.content_chars, 1),
log(paragraphs_without_refs_total_length + 1),
paragraphs_without_refs_total_length /
max(wikitext.revision.content_chars, 1),
]
wp10 = wikipedia.article + local_wiki