def test_chunk_id_to_filename(self):
# create regticket
regticket = ArtRegistrationClient.generate_regticket(
png_1x1_data, {
'artist_name': 'artist_name',
'artist_website': 'artist_website',
'artist_written_statement': 'artist_written_statement',
'artwork_title': 'artwork_title',
'artwork_series_name': 'artwork_series_name',
'artwork_creation_video_youtube_url':
'artwork_creation_video_youtube_url',
'artwork_keyword_set': 'artwork_keyword_set',
'total_copies': 99,
'copy_price': 555
})
# emulate what we do on MN0 when storing data to chunkstorage
imagedata = ImageData(
dictionary={
"image":
png_1x1_data,
"lubychunks":
ImageData.generate_luby_chunks(png_1x1_data,
seeds=regticket.lubyseeds),
"thumbnail":
ImageData.generate_thumbnail(png_1x1_data),
})
artwork_hash = imagedata.get_artwork_hash()
self.assertEqual(artwork_hash, regticket.imagedata_hash)
masternode_place_image_data_in_chunkstorage(regticket, png_1x1_data)
chunkids = list(get_chunkmanager().index_temp_storage())
# verify that our two chunks are actually in this chunksids list
# get image chunks IDs from regticket
image_chunkid = bytes_to_chunkid(
regticket.lubyhashes[0]
) # for very small image this is only 1 hash in list
# get thumbnail chunk ID from regticket
thumbnail_chunkid = bytes_to_chunkid(regticket.thumbnailhash)
self.assertIn(image_chunkid, chunkids)
self.assertIn(thumbnail_chunkid, chunkids)
# good. now we've stored image and thumbnail in tmpstorage.
# let's move it to regular storage
self.assertEqual(Chunk.select().count(),
2) # expect 2 chunks - image and thumbnail
self.assertEqual(
Chunk.select().where(Chunk.confirmed == False).count(),
2) # expect 2 chunks - image and thumbnail
# # set Chunk.confirmed = True as if we've processed activation ticket
Chunk.update(confirmed=True).execute()
move_confirmed_chunks_to_persistant_storage()
# try chunkstorage.get by chunk ID from regticket
# in regticket list of lubyhashes are chunks IDs.
image_chunk = get_chunkmanager().get_chunk_data(image_chunkid)
thmbnail_chunk = get_chunkmanager().get_chunk_data(thumbnail_chunkid)
self.assertIsNotNone(image_chunk)
print(len(image_chunk))
self.assertIsNotNone(thmbnail_chunk)
def test_task(self, get_blockchain_connection):
get_blockchain_connection(
).list_tickets.return_value = TICKETS_LIST_ACT_RESPONSE
get_blockchain_connection(
).get_ticket.side_effect = get_ticket_side_effect
self.assertEqual(Chunk.select().count(), 0)
self.assertEqual(ActivationTicket.select().count(), 0)
get_and_proccess_new_activation_tickets()
self.assertEqual(ActivationTicket.select().count(), 1)
self.assertEqual(Chunk.select().count(), 9)
def test_calculate_xor_distances_for_masternodes(
self, bc_obj, get_blockchain_connection):
bc_obj.return_value.masternode_list.return_value = mn_list
get_blockchain_connection.return_value.masternode_list.return_value = mn_list
for i in range(3):
Chunk.create(
chunk_id=
'1231231231231231232323934384834890089238429382938429384934{}'.
format(i),
image_hash=b'asdasdasd')
refresh_masternode_list()
self.assertEqual(Masternode.get_active_nodes().count(), 3)
self.assertEqual(ChunkMnDistance.select().count(), 9)
def setUp(self):
MASTERNODE_DB.init(':memory:')
MASTERNODE_DB.connect(reuse_if_open=True)
MASTERNODE_DB.create_tables(DB_MODELS)
for i in range(3):
Masternode.create(
ext_address='127.0.0.1:444{}'.format(i),
pastel_id=
'jXZVtBmehoxYPotVrLdByFNNcB8jsryXhFPgqRa95i2x1mknbzSef1oGjnzfiwRtzReimfugvg41VtA7qGfDZ{}'
.format(i))
Chunk.create(
chunk_id=
'1231231231231231232323934384834890089238429382938429384934{}'.
format(i),
image_hash=b'asdasdasd')
index_new_chunks()
def receive_rpc_fetchchunk(data, **kwargs):
if not isinstance(data, dict):
raise TypeError("Data must be a dict!")
if set(data.keys()) != {"chunkid"}:
raise ValueError("Invalid arguments for spotcheck: %s" % (data.keys()))
if not isinstance(data["chunkid"], str):
raise TypeError("Invalid type for key chunkid in spotcheck")
chunkid = hex_to_chunkid(
data["chunkid"]) # here chunkid is long integer number
# fetch chunk from DB, check if we store it
try:
chunk = Chunk.get(chunk_id=str(chunkid))
except peewee.DoesNotExist:
return {"chunk": None}
if not chunk.stored:
return {"chunk": None}
try:
chunk_data = get_chunkmanager().get_chunk_data(chunkid)
except FileNotFoundError:
chunk.stored = False
chunk.save()
chunk_data = None
return {"chunk": chunk_data}
def masternode_place_image_data_in_chunkstorage(regticket,
regticket_image_data):
"""
Place image to the chunkstorage. Initially artwork is placed to the so called temporary storage
(which exist only on the given masternode and is not distributed to other).
After activation ticket will be created (by the wallet) and parsed by current masternode -
masternode will move artwork chunks to regular chunkstorage and start promote chunks to
other masternodes, so artwork will be stored distributedly.
"""
# reconstruct image with seeds from regticket.
# then and only then chunk set will be identical to what wallet generated (and which hashes
# are written to the regticket.lubyhashes).
imagedata = ImageData(
dictionary={
"image":
regticket_image_data,
"lubychunks":
ImageData.generate_luby_chunks(regticket_image_data,
seeds=regticket.lubyseeds),
"thumbnail":
ImageData.generate_thumbnail(regticket_image_data),
})
artwork_hash = imagedata.get_artwork_hash()
# store thumbnail
get_chunkmanager().store_chunk_in_temp_storage(
bytes_to_chunkid(regticket.thumbnailhash), imagedata.thumbnail)
Chunk.create_from_hash(chunkhash=regticket.thumbnailhash,
artwork_hash=regticket.thumbnailhash,
stored=True)
# store chunks
for chunkhash, chunkdata in zip(imagedata.get_luby_hashes(),
imagedata.lubychunks):
chunkhash_int = bytes_to_chunkid(chunkhash)
get_chunkmanager().store_chunk_in_temp_storage(chunkhash_int,
chunkdata)
mn_ticket_logger.debug(
'Adding chunk id to DB: {}'.format(chunkhash_int))
# keep track of chunks in the local SQLite database.
# we should be ably to find all chunks by artwork hash as well.
# save chunk in database and mark `Stored` = True as we've already stored it in the storage (at least temp).
Chunk.create_from_hash(chunkhash=chunkhash,
artwork_hash=artwork_hash,
stored=True)
def get_chunk_owners(chunk_id):
"""
Return list of masternodes database objects who's expected to store a given chunk
"""
db_chunk = Chunk.get(chunk_id=str(chunk_id))
return [
c.masternode
for c in ChunkMnRanked.select().where(ChunkMnRanked.chunk == db_chunk)
]
def index_new_chunks():
"""
Select chunks which has not been indexed (XOR distance is not calculated) and calculate XOR distance for them.
"""
chunk_qs = Chunk.select().where(Chunk.indexed == False)
chunk_ids = [int(c.chunk_id) for c in chunk_qs]
if len(chunk_ids):
chunk_storage_logger.info(
'index_new_chunks found {} unprocessed chunks. Processing...'.
format(len(chunk_ids)))
calculate_xor_distances_for_chunks(chunk_ids)
# update all processed chunks with `indexed` = True
for chunk in chunk_qs:
chunk.indexed = True
Chunk.bulk_update(chunk_qs, fields=[Chunk.indexed])
chunk_storage_logger.info(
'Updating Chunk.indexed flag for processed chunks in DB')
# calculate chunk-mn-ranks as list of chunks was changed
recalculate_mn_chunk_ranking_table()
def receive_rpc_download_thumbnail(data, *args, **kwargs):
image_hash = data['image_hash']
chunks_db = Chunk.select().where(Chunk.image_hash == image_hash,
Chunk.stored == True)
if len(chunks_db) == 0:
# fixme: maybe if current MN does not stores a given thumbnail - it worth
# to return a list of masternodes which should store it (from ChunkMnRanked table).
try:
chunk = Chunk.select().get(Chunk.image_hash == image_hash)
except Exception:
return {
"status": "ERROR",
"msg": "No chunks for given image",
"masternodes": []
}
masternodes = [
c.masternode.pastel_id
for c in ChunkMnRanked.select().where(ChunkMnRanked.chunk == chunk)
]
return {
"status": "ERROR",
"msg": "No chunks for given image",
"masternodes": masternodes
}
if len(chunks_db) > 1:
return {
"status":
"ERROR",
"msg":
"There {} chunks for thumbnails in DB. should be one.".format(
len(chunks_db))
}
thumbnail_data = get_chunkmanager().get_chunk_data(
int(chunks_db[0].chunk_id))
# thumbnail is not encoded, so no decore is required.
return {"status": "SUCCESS", "image_data": thumbnail_data}
def calculate_xor_distances_for_masternodes(pastelids):
"""
`pastelids` - list of pastelids of masternodes. PastelID is a string.
"""
mns_db = Masternode.get_active_nodes().where(
Masternode.pastel_id.in_(pastelids))
for mn in mns_db:
for chunk in Chunk.select():
# we store chunk.chunk_id as CharField, but essentially it's very long integer (more then 8 bytes,
# doesn't fit in database INT type)
xor_distance = calculate_xor_distance(mn.pastel_id,
int(chunk.chunk_id))
ChunkMnDistance.create(chunk=chunk,
masternode=mn,
distance=str(xor_distance))
async def fetch_chunk_and_store_it(chunkid):
chunk = Chunk.get(Chunk.chunk_id == chunkid)
if chunk.attempts_to_load not in FIBONACHI_ROW:
chunk.attempts_to_load += 1
chunk.save()
return False
data = await fetch_single_chunk_via_rpc(chunkid)
if data:
# add chunk to persistant storage and update DB info (`stored` flag) to True
get_chunkmanager().store_chunk_in_storage(int(chunkid), data)
chunk.stored = True
chunk.attempts_to_load += 1
chunk.save()
return True
else:
chunk.attempts_to_load += 1
chunk.save()
return False
def test_xor_distance_task_with_2_chunks(self):
Chunk.create(
chunk_id=
'4529959709239007853998547086821683042815765622154307906125136018'
'25293195444578222995977844421809007120124095843869086665678514889'
'572116942841928123088049',
image_hash=b'123123123')
Chunk.create(
chunk_id=
'4529959709239007853998547086821683042815765622154307906125136018'
'25293195444578222995977844421809007120124095843869086665678514889'
'572116942841928123088041',
image_hash=b'123123123')
self.assertEqual(Chunk.select()[0].indexed, False)
index_new_chunks()
self.assertEqual(len(ChunkMnDistance.select()), 4)
self.assertEqual(Chunk.select()[0].indexed, True)
self.assertEqual(Chunk.select()[1].indexed, True)
def move_confirmed_chunks_to_persistant_storage():
"""
Task which moves chunks from temp storage to persistent one.
- Goes through chunks in temp storage,
- fetch each chunk from DB, if it's confirmed - move to persistant storage.
"""
for chunk_id in get_chunkmanager().index_temp_storage():
tasks_logger.warn('Process chunk {}'.format(chunk_id))
try:
chunk_db = Chunk.get(chunk_id=chunk_id)
except DoesNotExist:
tasks_logger.exception(
'Chunk with id {} does not exist in DB but exist in local storage'
.format(chunk_id))
get_chunkmanager().rm_from_temp_storage(chunk_id)
continue
if chunk_db.confirmed:
# move to persistant storge
tasks_logger.warn('Move chunk to persistant storage')
get_chunkmanager().move_to_persistant_storage(chunk_id)
def calculate_xor_distances_for_chunks(chunk_ids):
"""
`chunk_ids` - list of chunks ids. Chunk ID is a very long integer.
"""
chunk_storage_logger.info(
'Calculating XOR distance for {} chunks...'.format(len(chunk_ids)))
chunk_ids_str = [str(x) for x in chunk_ids]
chunks_db = Chunk.select().where(Chunk.chunk_id.in_(chunk_ids_str))
counter = 0
for chunk in chunks_db:
for mn in Masternode.get_active_nodes():
# we store chunk.chunk_id as CharField, but essentially it's very long integer (more then 8 bytes,
# doesn't fit in database INT type)
xor_distance = calculate_xor_distance(mn.pastel_id,
int(chunk.chunk_id))
ChunkMnDistance.create(chunk=chunk,
masternode=mn,
distance=str(xor_distance))
counter += 1
chunk_storage_logger.info('..Caculated {} distances'.format(counter))
def receive_rpc_download_image(data, *args, **kwargs):
# fixme: image download logic should be much more complex.
# - masternode receive image download request, generate unique code and return to the client
# - MN has to validate that requestor pastelID is artwork owner pastelID
# - if check passed - collect required chunks from other MNs
# - assemble image and store it locally
# - provide interface for client to poll if image is ready or not.
image_hash = data['image_hash']
chunks_db = Chunk.select().where(Chunk.image_hash == image_hash)
if len(chunks_db) == 0:
return {"status": "ERROR", "mgs": "No chunks for given image"}
chunks = [
get_chunkmanager().get_chunk_data(int(x.chunk_id)) for x in chunks_db
]
try:
image_data = luby.decode(chunks)
except luby.NotEnoughChunks:
return {
"status": "ERROR",
"mgs": "Not enough chunks to reconstruct given image"
}
return {"status": "SUCCESS", "image_data": image_data}
def test_get_missing_chunks_2(self):
pastel_id = 'jXZVtBmehoxYPotVrLdByFNNcB8jsryXhFPgqRa95i2x1mknbzSef1oGjnzfiwRtzReimfugvg41VtA7qGfDZ0'
Chunk.update(stored=True).where(Chunk.id == 1).execute()
recalculate_mn_chunk_ranking_table()
chunks = get_missing_chunk_ids(pastel_id)
self.assertEqual(len(chunks), 2)
def test_no_act_ticket(self, get_blockchain_connection):
get_blockchain_connection().list_tickets.return_value = []
get_and_proccess_new_activation_tickets()
self.assertEqual(ActivationTicket.select().count(), 0)
self.assertEqual(Chunk.select().count(), 0)
def get_and_proccess_new_activation_tickets():
"""
As as input we have list of new activation ticket.
Outputs of this task are:
- Store appropriate registration tickets in local DB, marking them confirmed
- Store chunks from these registration tickets in local DB, marking them confirmed.
(which will be used by another chunk-processing tasks).
"""
# FIXME: use `height` param when it will be implemented on cNode
act_tickets = get_blockchain_connection().list_tickets(
'act') # list if dicts with actticket data
act_tickets_txids = [ticket['txid'] for ticket in act_tickets]
if act_tickets_txids is None:
return
for txid in filter(lambda x: len(x) == TXID_LENGTH, act_tickets_txids):
if ActivationTicket.select().where(
ActivationTicket.txid == txid).count() != 0:
continue
tasks_logger.info('New activation ticket found: {}'.format(txid))
try:
act_ticket = get_blockchain_connection().get_ticket(txid)
except JSONRPCException as e:
tasks_logger.exception(
'Exception while fetching actticket: {}'.format(str(e)))
# to avoid processing invalid txid multiple times - write in to the DB with height=-1
ActivationTicket.create(txid=txid, height=-1)
continue
# fetch regticket from activation ticket
# store regticket in local DB if not exist
# get list of chunk ids, add to local DB (Chunk table)
regticket_data = get_blockchain_connection().get_ticket(
act_ticket['ticket']['reg_txid'])
regticket = get_registration_ticket_object_from_data(regticket_data)
chunk_hashes = regticket.lubyhashes # this is list of bytes objects
# add thumbnail chunk
try:
tasks_logger.info(
'Creating Chunk record for thumbnail, hash {}'.format(
regticket.thumbnailhash))
chunk = Chunk.create_from_hash(
chunkhash=regticket.thumbnailhash,
artwork_hash=regticket.thumbnailhash)
except IntegrityError: # if Chunk with such chunkhash already exist
tasks_logger.error('Error: thumbnail chunk already exists')
chunk = Chunk.get_by_hash(chunkhash=regticket.thumbnailhash)
chunk.confirmed = True
chunk.save()
chunks_created, chunks_updated = 0, 0
tasks_logger.info('Creating chunks record for artwork chunks...')
for chunkhash in chunk_hashes:
# if chunk exists - mark it as confirmed
# else - create it. And mark as confirmed. More frequently we'll have no such chunk.
try:
chunk = Chunk.create_from_hash(
chunkhash=chunkhash, artwork_hash=regticket.imagedata_hash)
chunks_created += 1
except IntegrityError: # if Chunk with such chunkhash already exist
chunk = Chunk.get_by_hash(chunkhash=chunkhash)
chunks_updated += 1
chunk.confirmed = True
chunk.save()
tasks_logger.info('...Complete! Created {}, updated {} chunks'.format(
chunks_created, chunks_updated))
# write processed act ticket to DB
tasks_logger.info(
'Activation ticket processed, writing to the DB. Height: {}'.
format(regticket_data['height']))
ActivationTicket.create(txid=txid, height=act_ticket['height'])
def test_xor_distance_task_without_chunks(self):
self.assertEqual(len(Chunk.select()), 0)
index_new_chunks()
self.assertEqual(len(ChunkMnDistance.select()), 0)
def test_place_in_chunkstorage(self, chunkstorage):
image_data = png_1x1_data
regticket = get_regticket()
masternode_place_image_data_in_chunkstorage(regticket, image_data)
self.assertEqual(len(os.listdir('tmpstorage')), 2)
self.assertEqual(Chunk.select().count(), 2)
def test_xor_distance_task_without_chunks_without_masternodes(self):
Masternode.delete()
self.assertEqual(len(Chunk.select()), 0)
index_new_chunks()
self.assertEqual(len(ChunkMnDistance.select()), 0)
