def test_order_of_signals_is_chronological(self):
with self.fresh_dynamodb():
table_manager = BanksTable(self.get_table(),
get_default_signal_type_mapping())
bank_id, bank_member_id = self._create_bank_and_bank_member()
signals = [
table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoMD5Signal,
signal_value="A VIDEO MD5 SIGNAL. WILTY?" +
str(random.random()),
) for _ in range(20)
]
signal_ids_in_order = list(
map(lambda s: s.signal_id,
sorted(signals, key=lambda x: x.updated_at)))
to_process_signal_ids = [
signal.signal_id for signal in
table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoMD5Signal).items
]
self.assertListEqual(signal_ids_in_order, to_process_signal_ids)
def test_bank_member_removes(self):
with self.fresh_dynamodb():
table_manager = BanksTable(self.get_table())
bank_id, bank_member_id = self._create_bank_and_bank_member()
bank_member_signal_1 = table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoTmkPdqfSignal,
signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?",
)
bank_member_signal_2 = table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoTmkPdqfSignal,
signal_value="ANOTHER VIDEO TMK PDQF SIGNAL. WILTY?",
)
bank_member_signal_3 = table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoTmkPdqfSignal,
signal_value="An ANOTHER VIDEO TMK PDQF SIGNAL. WILTY?",
)
# expect this to now be available to process
to_process = table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoTmkPdqfSignal)
self.assertEqual(len(to_process.items), 3)
table_manager.remove_bank_member_signals_to_process(
bank_member_id=bank_member_id)
# expect this to now be available to process
to_process = table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoTmkPdqfSignal)
self.assertEqual(len(to_process.items), 0)
def test_single_signal_is_retrieved(self):
with self.fresh_dynamodb():
table_manager = BanksTable(self.get_table())
bank_id, bank_member_id = self._create_bank_and_bank_member()
bank_member_signal = table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoTmkPdqfSignal,
signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?",
)
# expect this to now be available to process
to_process = table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoTmkPdqfSignal)
self.assertEqual(len(to_process.items), 1)
self.assertEqual(bank_member_signal.signal_id,
to_process.items[0].signal_id)
def test_multiple_signals_are_retrieved(self):
with self.fresh_dynamodb():
table_manager = BanksTable(self.get_table())
bank_id, bank_member_id = self._create_bank_and_bank_member()
signal_ids = [
table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoTmkPdqfSignal,
signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?" +
str(random.random()),
).signal_id for _ in range(20)
]
to_process_signal_ids = [
signal.signal_id for signal in
table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoTmkPdqfSignal).items
]
self.assertListEqual(signal_ids, to_process_signal_ids)
def test_order_of_signals_multi_page(self):
with self.fresh_dynamodb():
table_manager = BanksTable(self.get_table(),
get_default_signal_type_mapping())
bank_id, bank_member_id = self._create_bank_and_bank_member()
signals = [
table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoMD5Signal,
signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?" +
str(random.random()),
) for _ in range(20)
]
signal_ids_in_order = list(
map(lambda s: s.signal_id,
sorted(signals, key=lambda x: x.updated_at)))
queried_signal_ids = []
exclusive_start_key = None
while True:
response = table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoMD5Signal,
limit=4,
exclusive_start_key=exclusive_start_key,
)
exclusive_start_key = response.last_evaluated_key
queried_signal_ids += [
signal.signal_id for signal in response.items
]
if not response.has_next_page():
break
self.assertListEqual(signal_ids_in_order, queried_signal_ids)
def get_all_bank_hash_rows(signal_type: t.Type[SignalType],
banks_table: BanksTable) -> t.Iterable[HashRowT]:
"""
Make repeated calls to banks table to get all hashes for a signal type.
Returns list[HashRowT]. HashRowT is a tuple of hash_value and some metadata
about the signal.
"""
exclusive_start_key = None
hash_rows: t.List[HashRowT] = []
while True:
page = banks_table.get_bank_member_signals_to_process_page(
signal_type=signal_type, exclusive_start_key=exclusive_start_key)
for bank_member_signal in page.items:
hash_rows.append((
bank_member_signal.signal_value,
[
BankedSignalIndexMetadata(
bank_member_signal.signal_id,
bank_member_signal.signal_value,
bank_member_signal.bank_member_id,
),
],
))
exclusive_start_key = page.last_evaluated_key
if not page.has_next_page():
break
logger.info(
f"Obtained {len(hash_rows)} hash records from banks for signal_type:{signal_type.get_name()}"
)
return hash_rows
