def test_order_of_signals_is_chronological(self):
with self.fresh_dynamodb():
table_manager = BanksTable(self.get_table(),
get_default_signal_type_mapping())
bank_id, bank_member_id = self._create_bank_and_bank_member()
signals = [
table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoMD5Signal,
signal_value="A VIDEO MD5 SIGNAL. WILTY?" +
str(random.random()),
) for _ in range(20)
]
signal_ids_in_order = list(
map(lambda s: s.signal_id,
sorted(signals, key=lambda x: x.updated_at)))
to_process_signal_ids = [
signal.signal_id for signal in
table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoMD5Signal).items
]
self.assertListEqual(signal_ids_in_order, to_process_signal_ids)
def test_bank_member_removes(self):
with self.fresh_dynamodb():
table_manager = BanksTable(self.get_table())
bank_id, bank_member_id = self._create_bank_and_bank_member()
bank_member_signal_1 = table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoTmkPdqfSignal,
signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?",
)
bank_member_signal_2 = table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoTmkPdqfSignal,
signal_value="ANOTHER VIDEO TMK PDQF SIGNAL. WILTY?",
)
bank_member_signal_3 = table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoTmkPdqfSignal,
signal_value="An ANOTHER VIDEO TMK PDQF SIGNAL. WILTY?",
)
# expect this to now be available to process
to_process = table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoTmkPdqfSignal)
self.assertEqual(len(to_process.items), 3)
table_manager.remove_bank_member_signals_to_process(
bank_member_id=bank_member_id)
# expect this to now be available to process
to_process = table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoTmkPdqfSignal)
self.assertEqual(len(to_process.items), 0)
def test_single_signal_is_retrieved(self):
with self.fresh_dynamodb():
table_manager = BanksTable(self.get_table())
bank_id, bank_member_id = self._create_bank_and_bank_member()
bank_member_signal = table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoTmkPdqfSignal,
signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?",
)
# expect this to now be available to process
to_process = table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoTmkPdqfSignal)
self.assertEqual(len(to_process.items), 1)
self.assertEqual(bank_member_signal.signal_id,
to_process.items[0].signal_id)
def add_bank_member_signal(
banks_table: BanksTable,
bank_id: str,
bank_member_id: str,
signal_type: t.Type[SignalType],
signal_value: str,
) -> BankMemberSignal:
"""
Add a bank member signal. Will deduplicate a signal_value + signal_type
tuple before writing to the database.
Calling this API also makes the signal (new or existing) available to
process into matching indices.
"""
return banks_table.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=signal_type,
signal_value=signal_value,
)
def test_multiple_signals_are_retrieved(self):
with self.fresh_dynamodb():
table_manager = BanksTable(self.get_table())
bank_id, bank_member_id = self._create_bank_and_bank_member()
signal_ids = [
table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoTmkPdqfSignal,
signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?" +
str(random.random()),
).signal_id for _ in range(20)
]
to_process_signal_ids = [
signal.signal_id for signal in
table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoTmkPdqfSignal).items
]
self.assertListEqual(signal_ids, to_process_signal_ids)
def test_order_of_signals_multi_page(self):
with self.fresh_dynamodb():
table_manager = BanksTable(self.get_table(),
get_default_signal_type_mapping())
bank_id, bank_member_id = self._create_bank_and_bank_member()
signals = [
table_manager.add_bank_member_signal(
bank_id=bank_id,
bank_member_id=bank_member_id,
signal_type=VideoMD5Signal,
signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?" +
str(random.random()),
) for _ in range(20)
]
signal_ids_in_order = list(
map(lambda s: s.signal_id,
sorted(signals, key=lambda x: x.updated_at)))
queried_signal_ids = []
exclusive_start_key = None
while True:
response = table_manager.get_bank_member_signals_to_process_page(
signal_type=VideoMD5Signal,
limit=4,
exclusive_start_key=exclusive_start_key,
)
exclusive_start_key = response.last_evaluated_key
queried_signal_ids += [
signal.signal_id for signal in response.items
]
if not response.has_next_page():
break
self.assertListEqual(signal_ids_in_order, queried_signal_ids)
