def receive_rpc_fetchchunk(data, **kwargs):
if not isinstance(data, dict):
raise TypeError("Data must be a dict!")
if set(data.keys()) != {"chunkid"}:
raise ValueError("Invalid arguments for spotcheck: %s" % (data.keys()))
if not isinstance(data["chunkid"], str):
raise TypeError("Invalid type for key chunkid in spotcheck")
chunkid = hex_to_chunkid(
data["chunkid"]) # here chunkid is long integer number
# fetch chunk from DB, check if we store it
try:
chunk = Chunk.get(chunk_id=str(chunkid))
except peewee.DoesNotExist:
return {"chunk": None}
if not chunk.stored:
return {"chunk": None}
try:
chunk_data = get_chunkmanager().get_chunk_data(chunkid)
except FileNotFoundError:
chunk.stored = False
chunk.save()
chunk_data = None
return {"chunk": chunk_data}
def get_chunk_owners(chunk_id):
"""
Return list of masternodes database objects who's expected to store a given chunk
"""
db_chunk = Chunk.get(chunk_id=str(chunk_id))
return [
c.masternode
for c in ChunkMnRanked.select().where(ChunkMnRanked.chunk == db_chunk)
]
async def fetch_chunk_and_store_it(chunkid):
chunk = Chunk.get(Chunk.chunk_id == chunkid)
if chunk.attempts_to_load not in FIBONACHI_ROW:
chunk.attempts_to_load += 1
chunk.save()
return False
data = await fetch_single_chunk_via_rpc(chunkid)
if data:
# add chunk to persistant storage and update DB info (`stored` flag) to True
get_chunkmanager().store_chunk_in_storage(int(chunkid), data)
chunk.stored = True
chunk.attempts_to_load += 1
chunk.save()
return True
else:
chunk.attempts_to_load += 1
chunk.save()
return False
def move_confirmed_chunks_to_persistant_storage():
"""
Task which moves chunks from temp storage to persistent one.
- Goes through chunks in temp storage,
- fetch each chunk from DB, if it's confirmed - move to persistant storage.
"""
for chunk_id in get_chunkmanager().index_temp_storage():
tasks_logger.warn('Process chunk {}'.format(chunk_id))
try:
chunk_db = Chunk.get(chunk_id=chunk_id)
except DoesNotExist:
tasks_logger.exception(
'Chunk with id {} does not exist in DB but exist in local storage'
.format(chunk_id))
get_chunkmanager().rm_from_temp_storage(chunk_id)
continue
if chunk_db.confirmed:
# move to persistant storge
tasks_logger.warn('Move chunk to persistant storage')
get_chunkmanager().move_to_persistant_storage(chunk_id)