def test_where_on_entries_connector():
entries_connector = EntriesConnector(ENTRIES)
obtained = entries_connector.query(
Query(
filters = BinaryPredicate(
BinaryPredicate("a", "<=", 10),
"&&",
BinaryPredicate("c", ">", 0)
)
)
)
assert obtained == EXPECTED
def test_join_if_connector():
expected = {
INNER_JOIN : 1,
LEFT_JOIN : 2,
RIGHT_JOIN : 2,
FULL_OUTER_JOIN : 3
}
for mode in expected.keys():
connector = JoinIfConnector(
EntriesConnector(ENTRIES1),
EntriesConnector(ENTRIES2),
lambda l, r: l["k12"] == r["k12"],
mode
)
assert len(connector.query(Query())) == expected[mode]
def test_where_connector():
entries_connector = EntriesConnector(ENTRIES)
where_connector = WhereConnector(entries_connector, KEEP_IF)
obtained = where_connector.query(
Query()
)
assert obtained == EXPECTED
def test_query_select_where():
entries_connector = EntriesConnector(ENTRIES)
q = Query(attributes=["a", "c", "d"],
filters=BinaryPredicate(BinaryPredicate("a", "<=", 100), "&&",
BinaryPredicate("b", ">", 20)))
result = entries_connector.query(q)
assert result == [{
'a': 100,
'c': 300,
'd': None
}, {
'a': 100,
'c': None,
'd': 400
}]
def test_count_connector():
entries_connector = EntriesConnector(ENTRIES)
count_connector = CountConnector(entries_connector)
assert count_connector.query(Query()) == 4
assert count_connector.query(
Query(filters=BinaryPredicate("c", ">", 20))) == 2
assert count_connector.query(
Query(filters=BinaryPredicate("c", ">", 1000))) == 0
def test_download_select():
entries = [{"url": url} for url in URLS]
c = DownloadConnector({"url": "response"}, EntriesConnector(entries))
entries = c.query(Query(attributes=("url", "response")))
success = 0
for entry in entries:
if isinstance(entry["response"], str):
success += 1
assert success >= 1
def test_offset_limit():
entries_connector = EntriesConnector(ENTRIES)
attributes = ["a", "b", "c"]
for offset in range(len(ENTRIES)):
for limit in range(len(ENTRIES)):
q = Query(attributes=attributes, offset=offset, limit=limit)
Log.debug(q)
result = entries_connector.query(q)
Log.debug(pformat(result))
assert len(result) == min(limit, len(ENTRIES) - offset),\
"Invalid #entries for %s:\n%s" % (str(q), pformat(result))
expected = [
{k : entry.get(k) for k in attributes} \
for entry in ENTRIES[offset : offset + limit]
]
assert result == expected, """
Got : %s\n
Expected : %s\n
""" % (result, expected)
def test_group_by_connector():
print("connector group by b, a")
group_by_connector = GroupByConnector(("b", "a"),
EntriesConnector(ENTRIES))
assert len(
group_by_connector.query(
Query(
action=ACTION_READ,
object="",
attributes=[],
))) == 4
def test_unique_connector():
unique_connector = UniqueConnector(
["a"],
EntriesConnector(ENTRIES)
)
obtained = unique_connector.query(Query())
expected = [
{"a" : 1, "b" : 200, "c" : 31},
{"a" : 10, "b" : 200, "c" : 300},
{"a" : 100, "b" : 2, "c" : 30}
]
assert obtained == expected
def test_html_to_text():
urls = ["https://www.merriam-webster.com/dictionary/tapir"]
entries = [{"url": url} for url in urls]
c = DownloadConnector({"url": "response"}, EntriesConnector(entries))
entries = c.query(Query(attributes=("url", "response")))
success = 0
for entry in entries:
response = entry["response"]
if isinstance(entry["response"], str):
assert not "</" in response, "Found </ in %r" % response
success += 1
assert success >= 1
def test_download_connector():
entries = [{"url": url} for url in URLS]
c = DownloadConnector(
{"url": "response"
}, # Means entry["response"] = download(entry["url"])
EntriesConnector(entries))
entries = c.query(Query())
success = 0
for entry in entries:
if isinstance(entry["response"], str):
success += 1
assert success >= 1
def test_rename_select_where():
rename_connector = RenameConnector(MAP_RENAME, EntriesConnector(ENTRIES))
query = Query(attributes=["A", "C", "D"],
filters=BinaryPredicate(BinaryPredicate("A", "<=", 100),
"&&", BinaryPredicate("B", ">", 20)))
result = rename_connector.query(query)
assert result == [{
"A": 100,
"C": 300,
"D": None
}, {
"A": 100,
"C": None,
"D": 400
}]
def test_cache_lifetime():
short_lifetime = datetime.timedelta(milliseconds=50)
cache_connectors_short_lifetime = [
cls(EntriesConnector(ENTRIES), lifetime=short_lifetime) \
for cls in STORAGE_CONNECTOR_CLASSES
]
query = Query()
for cache_connector in cache_connectors_short_lifetime:
query = Query()
cache_connector.clear_query(query)
assert cache_connector.is_cached(query) == False
cache_connector.query(query)
assert cache_connector.is_cached(query) == True
sleep(short_lifetime.total_seconds())
assert cache_connector.is_cached(query) == False
def test_cache_rebase():
DUMMY_BASE_DIR = "/tmp/.minifold"
def check_base_dir(cache_connectors: list,
dummy_cache_connectors: list = list()):
query = Query()
for cache_connector in cache_connectors:
cache_filename = cache_connector.make_cache_filename(query)
Log.debug(cache_filename)
assert cache_filename.startswith(DEFAULT_CACHE_STORAGE_BASE_DIR)
for dummy_cache_connector in dummy_cache_connectors:
cache_filename = dummy_cache_connector.make_cache_filename(query)
Log.debug(cache_filename)
assert cache_filename.startswith(DUMMY_BASE_DIR)
# CACHE_CONNECTORS should be stored in DEFAULT_CACHE_STORAGE_BASE_DIR.
check_base_dir(CACHE_CONNECTORS, [])
# We now rebase the default cache directory to DUMMY_BASE_DIR.
# Caches newly created should be stored in DUMMY_BASE_DIR but the caches
# previously created should remain in their place.
Log.info("Setting StorageCacheConnector.base_dir to [%s]" % DUMMY_BASE_DIR)
StorageCacheConnector.base_dir = DUMMY_BASE_DIR
dummy_cache_connectors = [
cls(EntriesConnector(ENTRIES)) \
for cls in STORAGE_CONNECTOR_CLASSES
]
check_base_dir(CACHE_CONNECTORS, dummy_cache_connectors)
# We now rebase the default cache directory the standard cache directory.
# Caches newly created should be stored in DEFAULT_CACHE_STORAGE_BASE_DIR but the caches
# previously created should remain in their place.
Log.info("Setting StorageCacheConnector.base_dir to [%s]" %
DEFAULT_CACHE_STORAGE_BASE_DIR)
StorageCacheConnector.base_dir = DEFAULT_CACHE_STORAGE_BASE_DIR
check_base_dir(CACHE_CONNECTORS, dummy_cache_connectors)
def test_limit_connector():
entries_connector = EntriesConnector(ENTRIES)
limit_connector = LimitConnector(entries_connector, 2)
obtained = limit_connector.query(Query())
assert obtained == EXPECTED
def test_select_connector_alters_query():
entries_connector = EntriesConnector(ENTRIES)
select_connector = SelectConnector(entries_connector, ATTRIBUTES)
obtained = select_connector.query(Query(attributes=[]))
assert obtained == EXPECTED
def test_select_on_entries_connector():
entries_connector = EntriesConnector(ENTRIES)
obtained = entries_connector.query(Query(attributes=ATTRIBUTES))
assert obtained == EXPECTED
'b': 20,
'c': 30
},
{
'a': 100,
'b': 200,
'c': 300
},
{
'a': 100,
'b': 200,
'd': 400
},
]
ENTRIES_CONNECTOR = EntriesConnector(ENTRIES)
EMPTY_CONNECTOR = EntriesConnector([])
def test_union():
assert union([ENTRIES, []]) == ENTRIES
assert union([[], ENTRIES]) == ENTRIES
assert union([ENTRIES, ENTRIES]) == ENTRIES + ENTRIES
def test_union_connector_empty():
union_connector = UnionConnector([ENTRIES_CONNECTOR, EMPTY_CONNECTOR])
union_connector.query(Query()) == ENTRIES
def test_union_connector_duplicates():
"c": 30
},
{
"a": 100,
"b": 200,
"c": 300
},
{
"a": 100,
"b": 200,
"d": 400
},
]
CACHE_CONNECTORS = [
PickleCacheConnector(EntriesConnector(ENTRIES)),
JsonCacheConnector(EntriesConnector(ENTRIES))
]
SHORT_LIFETIME = datetime.timedelta(milliseconds=50)
CACHE_CONNECTORS_SHORT_LIFETIME = [
PickleCacheConnector(EntriesConnector(ENTRIES), lifetime=SHORT_LIFETIME),
JsonCacheConnector(EntriesConnector(ENTRIES), lifetime=SHORT_LIFETIME)
]
def test_clear_query():
for cache_connector in CACHE_CONNECTORS:
query = Query()
cache_connector.clear_query(query)
},
{
"a": 100,
"b": 200,
"c": 300
},
{
"a": 100,
"b": 200,
"d": 400
},
]
STORAGE_CONNECTOR_CLASSES = [PickleCacheConnector, JsonCacheConnector]
CACHE_CONNECTORS = [
cls(EntriesConnector(ENTRIES)) \
for cls in STORAGE_CONNECTOR_CLASSES
]
def test_clear_query():
for cache_connector in CACHE_CONNECTORS:
query = Query()
cache_connector.clear_query(query)
assert cache_connector.is_cached(query) == False
def test_clear_cache():
queries = [Query(), Query(attributes=("a", ))]
for cache_connector in CACHE_CONNECTORS:
for query in queries:
ENTRIES = [
{"a" : 1, "b" : 2, "c" : 3},
{"a" : 10, "b" : 20, "c" : 30},
{"a" : 100, "b" : 200, "c" : 300},
{"a" : 100, "b" : 200, "d" : 400},
]
MAP_RENAME = {
"a" : "A",
"c" : "C",
"d" : "D",
}
RENAME_CONNECTOR = RenameConnector(
MAP_RENAME,
EntriesConnector(ENTRIES)
)
def test_rename_select_where():
query = Query(
attributes = ["A", "C", "D"],
filters = BinaryPredicate(BinaryPredicate("A", "<=", 100), "&&", BinaryPredicate("b", ">", 20))
)
obtained = RENAME_CONNECTOR.query(query)
assert obtained == [
{"A": 100, "C": 300, "D": None},
{"A": 100, "C": None, "D": 400}
]
def test_rename_attributes():