def test_order_of_signals_is_chronological(self): with self.fresh_dynamodb(): table_manager = BanksTable(self.get_table(), get_default_signal_type_mapping()) bank_id, bank_member_id = self._create_bank_and_bank_member() signals = [ table_manager.add_bank_member_signal( bank_id=bank_id, bank_member_id=bank_member_id, signal_type=VideoMD5Signal, signal_value="A VIDEO MD5 SIGNAL. WILTY?" + str(random.random()), ) for _ in range(20) ] signal_ids_in_order = list( map(lambda s: s.signal_id, sorted(signals, key=lambda x: x.updated_at))) to_process_signal_ids = [ signal.signal_id for signal in table_manager.get_bank_member_signals_to_process_page( signal_type=VideoMD5Signal).items ] self.assertListEqual(signal_ids_in_order, to_process_signal_ids)
def test_bank_member_removes(self): with self.fresh_dynamodb(): table_manager = BanksTable(self.get_table()) bank_id, bank_member_id = self._create_bank_and_bank_member() bank_member_signal_1 = table_manager.add_bank_member_signal( bank_id=bank_id, bank_member_id=bank_member_id, signal_type=VideoTmkPdqfSignal, signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?", ) bank_member_signal_2 = table_manager.add_bank_member_signal( bank_id=bank_id, bank_member_id=bank_member_id, signal_type=VideoTmkPdqfSignal, signal_value="ANOTHER VIDEO TMK PDQF SIGNAL. WILTY?", ) bank_member_signal_3 = table_manager.add_bank_member_signal( bank_id=bank_id, bank_member_id=bank_member_id, signal_type=VideoTmkPdqfSignal, signal_value="An ANOTHER VIDEO TMK PDQF SIGNAL. WILTY?", ) # expect this to now be available to process to_process = table_manager.get_bank_member_signals_to_process_page( signal_type=VideoTmkPdqfSignal) self.assertEqual(len(to_process.items), 3) table_manager.remove_bank_member_signals_to_process( bank_member_id=bank_member_id) # expect this to now be available to process to_process = table_manager.get_bank_member_signals_to_process_page( signal_type=VideoTmkPdqfSignal) self.assertEqual(len(to_process.items), 0)
def test_single_signal_is_retrieved(self): with self.fresh_dynamodb(): table_manager = BanksTable(self.get_table()) bank_id, bank_member_id = self._create_bank_and_bank_member() bank_member_signal = table_manager.add_bank_member_signal( bank_id=bank_id, bank_member_id=bank_member_id, signal_type=VideoTmkPdqfSignal, signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?", ) # expect this to now be available to process to_process = table_manager.get_bank_member_signals_to_process_page( signal_type=VideoTmkPdqfSignal) self.assertEqual(len(to_process.items), 1) self.assertEqual(bank_member_signal.signal_id, to_process.items[0].signal_id)
def add_bank_member_signal( banks_table: BanksTable, bank_id: str, bank_member_id: str, signal_type: t.Type[SignalType], signal_value: str, ) -> BankMemberSignal: """ Add a bank member signal. Will deduplicate a signal_value + signal_type tuple before writing to the database. Calling this API also makes the signal (new or existing) available to process into matching indices. """ return banks_table.add_bank_member_signal( bank_id=bank_id, bank_member_id=bank_member_id, signal_type=signal_type, signal_value=signal_value, )
def test_multiple_signals_are_retrieved(self): with self.fresh_dynamodb(): table_manager = BanksTable(self.get_table()) bank_id, bank_member_id = self._create_bank_and_bank_member() signal_ids = [ table_manager.add_bank_member_signal( bank_id=bank_id, bank_member_id=bank_member_id, signal_type=VideoTmkPdqfSignal, signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?" + str(random.random()), ).signal_id for _ in range(20) ] to_process_signal_ids = [ signal.signal_id for signal in table_manager.get_bank_member_signals_to_process_page( signal_type=VideoTmkPdqfSignal).items ] self.assertListEqual(signal_ids, to_process_signal_ids)
def test_order_of_signals_multi_page(self): with self.fresh_dynamodb(): table_manager = BanksTable(self.get_table(), get_default_signal_type_mapping()) bank_id, bank_member_id = self._create_bank_and_bank_member() signals = [ table_manager.add_bank_member_signal( bank_id=bank_id, bank_member_id=bank_member_id, signal_type=VideoMD5Signal, signal_value="A VIDEO TMK PDQF SIGNAL. WILTY?" + str(random.random()), ) for _ in range(20) ] signal_ids_in_order = list( map(lambda s: s.signal_id, sorted(signals, key=lambda x: x.updated_at))) queried_signal_ids = [] exclusive_start_key = None while True: response = table_manager.get_bank_member_signals_to_process_page( signal_type=VideoMD5Signal, limit=4, exclusive_start_key=exclusive_start_key, ) exclusive_start_key = response.last_evaluated_key queried_signal_ids += [ signal.signal_id for signal in response.items ] if not response.has_next_page(): break self.assertListEqual(signal_ids_in_order, queried_signal_ids)