def test_insert_search_block(self, db): metadata = SearchMetadata( search_index_type='ExhaustiveSearchIndex', starting_search_index=ExhaustiveSearchIndex(n=1), ending_search_index=ExhaustiveSearchIndex(n=2), ) db.insert_search_blocks([metadata]) actual = db.claim_next_search_block( search_index_type='ExhaustiveSearchIndex') assert actual.state == SearchBlockState.IN_PROGRESS assert actual.key() == metadata.key() # no remaining unstarted blocks with pytest.raises(ValueError): db.claim_next_search_block( search_index_type='ExhaustiveSearchIndex')
def generate_next_block( self, batch_size: int) -> SearchMetadata[SuperabundantEnumerationIndex]: starting_index = deepcopy(self._search_index) ''' Because a block can cross multiple levels, we need to possibly split the block into pieces. ''' ending_level_index = self._search_index.index_in_level + batch_size - 1 while ending_level_index >= len(self.current_level): ending_level_index -= len(self.current_level) self._search_index = SuperabundantEnumerationIndex( level=self._search_index.level + 1, index_in_level=0) self.__maybe_reset_current_level__() # definitely ending_index = SuperabundantEnumerationIndex( level=self._search_index.level, index_in_level=ending_level_index) if (ending_level_index == len(self.current_level) - 1): self._search_index = SuperabundantEnumerationIndex( level=self._search_index.level + 1, index_in_level=0) self.__maybe_reset_current_level__() # definitely else: self._search_index = SuperabundantEnumerationIndex( level=self._search_index.level, index_in_level=ending_level_index + 1) return SearchMetadata( starting_search_index=starting_index, ending_search_index=ending_index, search_index_type='SuperabundantEnumerationIndex', )
def latest_search_metadata( self, search_state_type: str) -> Union[SearchMetadata, None]: cursor = self.connection.cursor() cursor.execute( ''' SELECT start_time, end_time, search_state_type, starting_search_state, ending_search_state FROM SearchMetadata WHERE search_state_type = :search_state_type ORDER BY end_time DESC LIMIT 1; ''', {'search_state_type': search_state_type}) row = cursor.fetchone() if not row: return None return SearchMetadata( start_time=row[0], end_time=row[1], search_state_type=row[2], starting_search_state=deserialize_search_state( search_state_type, row[3]), ending_search_state=deserialize_search_state( search_state_type, row[4]), )
def populate_search_blocks(self, db): metadatas = [ SearchMetadata(search_index_type='ExhaustiveSearchIndex', starting_search_index=ExhaustiveSearchIndex(n=i), ending_search_index=ExhaustiveSearchIndex(n=i + 1)) for i in range(1, 10, 2) ] db.insert_search_blocks(metadatas)
def test_claim_search_block_timestamp_updated(self, db): metadata = SearchMetadata( search_index_type='ExhaustiveSearchIndex', starting_search_index=ExhaustiveSearchIndex(n=1), ending_search_index=ExhaustiveSearchIndex(n=2), ) db.insert_search_blocks([metadata]) actual = db.claim_next_search_block( search_index_type='ExhaustiveSearchIndex') assert actual.start_time != None
def claim_next_search_block(self, search_index_type: str) -> SearchMetadata: cursor = self.connection.cursor() # FOR UPDATE locks the row for the duration of the query. # Cf. https://stackoverflow.com/q/11532550/438830 # and test_multiple_processors_no_duplicates cursor.execute(''' UPDATE SearchMetadata SET start_time = NOW(), state = 'IN_PROGRESS' FROM ( SELECT search_index_type, starting_search_index, ending_search_index FROM SearchMetadata WHERE search_index_type='%s' AND (state = 'NOT_STARTED' OR state = 'FAILED') ORDER BY creation_time ASC LIMIT 1 FOR UPDATE ) as m WHERE SearchMetadata.search_index_type = m.search_index_type AND SearchMetadata.starting_search_index = m.starting_search_index AND SearchMetadata.ending_search_index = m.ending_search_index RETURNING SearchMetadata.starting_search_index, SearchMetadata.ending_search_index, SearchMetadata.search_index_type, SearchMetadata.start_time, SearchMetadata.state ; ''' % search_index_type) if cursor.rowcount <= 0: raise ValueError('No legal search block to claim') row = cursor.fetchone() self.connection.commit() return SearchMetadata( starting_search_index=deserialize_search_index( search_index_type, row[0]), ending_search_index=deserialize_search_index( search_index_type, row[1]), search_index_type=row[2], start_time=row[3], # indexing [ ] is Python's "name to enum" lookup state=SearchBlockState[row[4]], )
def convert_metadatas(self, rows): return [ SearchMetadata( starting_search_index=deserialize_search_index(row[2], row[0]), ending_search_index=deserialize_search_index(row[2], row[1]), search_index_type=row[2], # indexing [ ] is Python's "name to enum" lookup state=SearchBlockState[row[3]], creation_time=row[4], start_time=row[5], end_time=row[6], block_hash=row[7], ) for row in rows ]
def test_store_metadata_ordering(self, newDatabase): name = "ExhaustiveSearchIndex" db: SearchMetadataDb = newDatabase() older_time = datetime(2016, 9, 1, 1, 0, 0) newer_time = datetime(2020, 9, 1, 1, 0, 0) metadata = [ SearchMetadata(start_time=older_time, end_time=older_time, search_state_type=name, starting_search_state=ExhaustiveSearchIndex(n=1), ending_search_state=ExhaustiveSearchIndex(n=2)), SearchMetadata(start_time=newer_time, end_time=newer_time, search_state_type=name, starting_search_state=ExhaustiveSearchIndex(n=1), ending_search_state=ExhaustiveSearchIndex(n=2)), ] # insert the newer one first db.insert_search_metadata(metadata[1]) db.insert_search_metadata(metadata[0]) assert metadata[1] == db.latest_search_metadata(name) db.teardown()
def test_store_metadata(self, newDatabase): name = "ExhaustiveSearchIndex" db: SearchMetadataDb = newDatabase() metadata = [ SearchMetadata(start_time=datetime(2020, 9, 1, 0, 0, 0), end_time=datetime(2020, 9, 1, 1, 0, 0), search_state_type=name, starting_search_state=ExhaustiveSearchIndex(n=1), ending_search_state=ExhaustiveSearchIndex(n=2)), ] db.insert_search_metadata(metadata[0]) assert metadata[0] == db.latest_search_metadata(name) db.teardown()
def generate_search_blocks( self, count: int, batch_size: int) -> List[SearchMetadata[ExhaustiveSearchIndex]]: start = self._search_index.n blocks = [] for i in range(count): starting_index = ExhaustiveSearchIndex(n=start) ending_index = ExhaustiveSearchIndex(n=start + batch_size - 1) start += batch_size blocks.append( SearchMetadata( starting_search_index=starting_index, ending_search_index=ending_index, search_index_type=self.index_name(), )) return blocks
def populate_db(divisorDb: DivisorDb, metadataDb: SearchMetadataDb, search_strategy: SearchStrategy, batch_size: int) -> None: '''Populate the db in batches. Write the computed divisor sums to the database after each batch. ''' while True: start = datetime.now() start_state = search_strategy.search_state() db.upsert(search_strategy.next_batch(batch_size)) end_state = search_strategy.search_state() end = datetime.now() print(f"Computed [{start_state.serialize()}, {end_state.serialize()}] in {end-start}") metadataDb.insert_search_metadata( SearchMetadata( start_time=start, end_time=end, search_state_type=end_state.__class__.__name__, starting_search_state=start_state, ending_search_state=end_state))
def test_search_block_with_mpz_values(self, db): metadatas = [ SearchMetadata(search_index_type='SuperabundantEnumerationIndex', starting_search_index=SuperabundantEnumerationIndex( level=i, index_in_level=0), ending_search_index=SuperabundantEnumerationIndex( level=i + 1, index_in_level=0)) for i in range(1, 10, 2) ] db.insert_search_blocks(metadatas) block = db.claim_next_search_block( search_index_type='SuperabundantEnumerationIndex') records = [ RiemannDivisorSum(n=mpz(1), divisor_sum=mpz(1), witness_value=1), RiemannDivisorSum(n=mpz(2), divisor_sum=mpz(2), witness_value=2), ] db.finish_search_block(block, records) stored = list(db.load()) assert len(stored) == 1 assert stored[0].n == 2