def test_insert_search_block(self, db):
metadata = SearchMetadata(
search_index_type='ExhaustiveSearchIndex',
starting_search_index=ExhaustiveSearchIndex(n=1),
ending_search_index=ExhaustiveSearchIndex(n=2),
)
db.insert_search_blocks([metadata])
actual = db.claim_next_search_block(
search_index_type='ExhaustiveSearchIndex')
assert actual.state == SearchBlockState.IN_PROGRESS
assert actual.key() == metadata.key()
# no remaining unstarted blocks
with pytest.raises(ValueError):
db.claim_next_search_block(
search_index_type='ExhaustiveSearchIndex')
def generate_next_block(
self,
batch_size: int) -> SearchMetadata[SuperabundantEnumerationIndex]:
starting_index = deepcopy(self._search_index)
'''
Because a block can cross multiple levels, we need to possibly split
the block into pieces.
'''
ending_level_index = self._search_index.index_in_level + batch_size - 1
while ending_level_index >= len(self.current_level):
ending_level_index -= len(self.current_level)
self._search_index = SuperabundantEnumerationIndex(
level=self._search_index.level + 1, index_in_level=0)
self.__maybe_reset_current_level__() # definitely
ending_index = SuperabundantEnumerationIndex(
level=self._search_index.level, index_in_level=ending_level_index)
if (ending_level_index == len(self.current_level) - 1):
self._search_index = SuperabundantEnumerationIndex(
level=self._search_index.level + 1, index_in_level=0)
self.__maybe_reset_current_level__() # definitely
else:
self._search_index = SuperabundantEnumerationIndex(
level=self._search_index.level,
index_in_level=ending_level_index + 1)
return SearchMetadata(
starting_search_index=starting_index,
ending_search_index=ending_index,
search_index_type='SuperabundantEnumerationIndex',
)
def latest_search_metadata(
self, search_state_type: str) -> Union[SearchMetadata, None]:
cursor = self.connection.cursor()
cursor.execute(
'''
SELECT
start_time,
end_time,
search_state_type,
starting_search_state,
ending_search_state
FROM SearchMetadata
WHERE search_state_type = :search_state_type
ORDER BY end_time DESC
LIMIT 1;
''', {'search_state_type': search_state_type})
row = cursor.fetchone()
if not row:
return None
return SearchMetadata(
start_time=row[0],
end_time=row[1],
search_state_type=row[2],
starting_search_state=deserialize_search_state(
search_state_type, row[3]),
ending_search_state=deserialize_search_state(
search_state_type, row[4]),
)
def populate_search_blocks(self, db):
metadatas = [
SearchMetadata(search_index_type='ExhaustiveSearchIndex',
starting_search_index=ExhaustiveSearchIndex(n=i),
ending_search_index=ExhaustiveSearchIndex(n=i + 1))
for i in range(1, 10, 2)
]
db.insert_search_blocks(metadatas)
def test_claim_search_block_timestamp_updated(self, db):
metadata = SearchMetadata(
search_index_type='ExhaustiveSearchIndex',
starting_search_index=ExhaustiveSearchIndex(n=1),
ending_search_index=ExhaustiveSearchIndex(n=2),
)
db.insert_search_blocks([metadata])
actual = db.claim_next_search_block(
search_index_type='ExhaustiveSearchIndex')
assert actual.start_time != None
def claim_next_search_block(self,
search_index_type: str) -> SearchMetadata:
cursor = self.connection.cursor()
# FOR UPDATE locks the row for the duration of the query.
# Cf. https://stackoverflow.com/q/11532550/438830
# and test_multiple_processors_no_duplicates
cursor.execute('''
UPDATE SearchMetadata
SET
start_time = NOW(),
state = 'IN_PROGRESS'
FROM (
SELECT
search_index_type,
starting_search_index,
ending_search_index
FROM SearchMetadata
WHERE
search_index_type='%s'
AND (state = 'NOT_STARTED' OR state = 'FAILED')
ORDER BY creation_time ASC
LIMIT 1
FOR UPDATE
) as m
WHERE
SearchMetadata.search_index_type = m.search_index_type
AND SearchMetadata.starting_search_index = m.starting_search_index
AND SearchMetadata.ending_search_index = m.ending_search_index
RETURNING
SearchMetadata.starting_search_index,
SearchMetadata.ending_search_index,
SearchMetadata.search_index_type,
SearchMetadata.start_time,
SearchMetadata.state
;
''' % search_index_type)
if cursor.rowcount <= 0:
raise ValueError('No legal search block to claim')
row = cursor.fetchone()
self.connection.commit()
return SearchMetadata(
starting_search_index=deserialize_search_index(
search_index_type, row[0]),
ending_search_index=deserialize_search_index(
search_index_type, row[1]),
search_index_type=row[2],
start_time=row[3],
# indexing [ ] is Python's "name to enum" lookup
state=SearchBlockState[row[4]],
)
def convert_metadatas(self, rows):
return [
SearchMetadata(
starting_search_index=deserialize_search_index(row[2], row[0]),
ending_search_index=deserialize_search_index(row[2], row[1]),
search_index_type=row[2],
# indexing [ ] is Python's "name to enum" lookup
state=SearchBlockState[row[3]],
creation_time=row[4],
start_time=row[5],
end_time=row[6],
block_hash=row[7],
) for row in rows
]
def test_store_metadata_ordering(self, newDatabase):
name = "ExhaustiveSearchIndex"
db: SearchMetadataDb = newDatabase()
older_time = datetime(2016, 9, 1, 1, 0, 0)
newer_time = datetime(2020, 9, 1, 1, 0, 0)
metadata = [
SearchMetadata(start_time=older_time,
end_time=older_time,
search_state_type=name,
starting_search_state=ExhaustiveSearchIndex(n=1),
ending_search_state=ExhaustiveSearchIndex(n=2)),
SearchMetadata(start_time=newer_time,
end_time=newer_time,
search_state_type=name,
starting_search_state=ExhaustiveSearchIndex(n=1),
ending_search_state=ExhaustiveSearchIndex(n=2)),
]
# insert the newer one first
db.insert_search_metadata(metadata[1])
db.insert_search_metadata(metadata[0])
assert metadata[1] == db.latest_search_metadata(name)
db.teardown()
def test_store_metadata(self, newDatabase):
name = "ExhaustiveSearchIndex"
db: SearchMetadataDb = newDatabase()
metadata = [
SearchMetadata(start_time=datetime(2020, 9, 1, 0, 0, 0),
end_time=datetime(2020, 9, 1, 1, 0, 0),
search_state_type=name,
starting_search_state=ExhaustiveSearchIndex(n=1),
ending_search_state=ExhaustiveSearchIndex(n=2)),
]
db.insert_search_metadata(metadata[0])
assert metadata[0] == db.latest_search_metadata(name)
db.teardown()
def generate_search_blocks(
self, count: int,
batch_size: int) -> List[SearchMetadata[ExhaustiveSearchIndex]]:
start = self._search_index.n
blocks = []
for i in range(count):
starting_index = ExhaustiveSearchIndex(n=start)
ending_index = ExhaustiveSearchIndex(n=start + batch_size - 1)
start += batch_size
blocks.append(
SearchMetadata(
starting_search_index=starting_index,
ending_search_index=ending_index,
search_index_type=self.index_name(),
))
return blocks
def populate_db(divisorDb: DivisorDb, metadataDb: SearchMetadataDb,
search_strategy: SearchStrategy, batch_size: int) -> None:
'''Populate the db in batches.
Write the computed divisor sums to the database after each batch.
'''
while True:
start = datetime.now()
start_state = search_strategy.search_state()
db.upsert(search_strategy.next_batch(batch_size))
end_state = search_strategy.search_state()
end = datetime.now()
print(f"Computed [{start_state.serialize()}, {end_state.serialize()}] in {end-start}")
metadataDb.insert_search_metadata(
SearchMetadata(
start_time=start,
end_time=end,
search_state_type=end_state.__class__.__name__,
starting_search_state=start_state,
ending_search_state=end_state))
def test_search_block_with_mpz_values(self, db):
metadatas = [
SearchMetadata(search_index_type='SuperabundantEnumerationIndex',
starting_search_index=SuperabundantEnumerationIndex(
level=i, index_in_level=0),
ending_search_index=SuperabundantEnumerationIndex(
level=i + 1, index_in_level=0))
for i in range(1, 10, 2)
]
db.insert_search_blocks(metadatas)
block = db.claim_next_search_block(
search_index_type='SuperabundantEnumerationIndex')
records = [
RiemannDivisorSum(n=mpz(1), divisor_sum=mpz(1), witness_value=1),
RiemannDivisorSum(n=mpz(2), divisor_sum=mpz(2), witness_value=2),
]
db.finish_search_block(block, records)
stored = list(db.load())
assert len(stored) == 1
assert stored[0].n == 2