def test_chunk_id_to_filename(self):
# create regticket
regticket = ArtRegistrationClient.generate_regticket(
png_1x1_data, {
'artist_name': 'artist_name',
'artist_website': 'artist_website',
'artist_written_statement': 'artist_written_statement',
'artwork_title': 'artwork_title',
'artwork_series_name': 'artwork_series_name',
'artwork_creation_video_youtube_url':
'artwork_creation_video_youtube_url',
'artwork_keyword_set': 'artwork_keyword_set',
'total_copies': 99,
'copy_price': 555
})
# emulate what we do on MN0 when storing data to chunkstorage
imagedata = ImageData(
dictionary={
"image":
png_1x1_data,
"lubychunks":
ImageData.generate_luby_chunks(png_1x1_data,
seeds=regticket.lubyseeds),
"thumbnail":
ImageData.generate_thumbnail(png_1x1_data),
})
artwork_hash = imagedata.get_artwork_hash()
self.assertEqual(artwork_hash, regticket.imagedata_hash)
masternode_place_image_data_in_chunkstorage(regticket, png_1x1_data)
chunkids = list(get_chunkmanager().index_temp_storage())
# verify that our two chunks are actually in this chunksids list
# get image chunks IDs from regticket
image_chunkid = bytes_to_chunkid(
regticket.lubyhashes[0]
) # for very small image this is only 1 hash in list
# get thumbnail chunk ID from regticket
thumbnail_chunkid = bytes_to_chunkid(regticket.thumbnailhash)
self.assertIn(image_chunkid, chunkids)
self.assertIn(thumbnail_chunkid, chunkids)
# good. now we've stored image and thumbnail in tmpstorage.
# let's move it to regular storage
self.assertEqual(Chunk.select().count(),
2) # expect 2 chunks - image and thumbnail
self.assertEqual(
Chunk.select().where(Chunk.confirmed == False).count(),
2) # expect 2 chunks - image and thumbnail
# # set Chunk.confirmed = True as if we've processed activation ticket
Chunk.update(confirmed=True).execute()
move_confirmed_chunks_to_persistant_storage()
# try chunkstorage.get by chunk ID from regticket
# in regticket list of lubyhashes are chunks IDs.
image_chunk = get_chunkmanager().get_chunk_data(image_chunkid)
thmbnail_chunk = get_chunkmanager().get_chunk_data(thumbnail_chunkid)
self.assertIsNotNone(image_chunk)
print(len(image_chunk))
self.assertIsNotNone(thmbnail_chunk)
def test_task(self, get_blockchain_connection):
get_blockchain_connection(
).list_tickets.return_value = TICKETS_LIST_ACT_RESPONSE
get_blockchain_connection(
).get_ticket.side_effect = get_ticket_side_effect
self.assertEqual(Chunk.select().count(), 0)
self.assertEqual(ActivationTicket.select().count(), 0)
get_and_proccess_new_activation_tickets()
self.assertEqual(ActivationTicket.select().count(), 1)
self.assertEqual(Chunk.select().count(), 9)
def test_xor_distance_task_with_2_chunks(self):
Chunk.create(
chunk_id=
'4529959709239007853998547086821683042815765622154307906125136018'
'25293195444578222995977844421809007120124095843869086665678514889'
'572116942841928123088049',
image_hash=b'123123123')
Chunk.create(
chunk_id=
'4529959709239007853998547086821683042815765622154307906125136018'
'25293195444578222995977844421809007120124095843869086665678514889'
'572116942841928123088041',
image_hash=b'123123123')
self.assertEqual(Chunk.select()[0].indexed, False)
index_new_chunks()
self.assertEqual(len(ChunkMnDistance.select()), 4)
self.assertEqual(Chunk.select()[0].indexed, True)
self.assertEqual(Chunk.select()[1].indexed, True)
def receive_rpc_download_thumbnail(data, *args, **kwargs):
image_hash = data['image_hash']
chunks_db = Chunk.select().where(Chunk.image_hash == image_hash,
Chunk.stored == True)
if len(chunks_db) == 0:
# fixme: maybe if current MN does not stores a given thumbnail - it worth
# to return a list of masternodes which should store it (from ChunkMnRanked table).
try:
chunk = Chunk.select().get(Chunk.image_hash == image_hash)
except Exception:
return {
"status": "ERROR",
"msg": "No chunks for given image",
"masternodes": []
}
masternodes = [
c.masternode.pastel_id
for c in ChunkMnRanked.select().where(ChunkMnRanked.chunk == chunk)
]
return {
"status": "ERROR",
"msg": "No chunks for given image",
"masternodes": masternodes
}
if len(chunks_db) > 1:
return {
"status":
"ERROR",
"msg":
"There {} chunks for thumbnails in DB. should be one.".format(
len(chunks_db))
}
thumbnail_data = get_chunkmanager().get_chunk_data(
int(chunks_db[0].chunk_id))
# thumbnail is not encoded, so no decore is required.
return {"status": "SUCCESS", "image_data": thumbnail_data}
def calculate_xor_distances_for_masternodes(pastelids):
"""
`pastelids` - list of pastelids of masternodes. PastelID is a string.
"""
mns_db = Masternode.get_active_nodes().where(
Masternode.pastel_id.in_(pastelids))
for mn in mns_db:
for chunk in Chunk.select():
# we store chunk.chunk_id as CharField, but essentially it's very long integer (more then 8 bytes,
# doesn't fit in database INT type)
xor_distance = calculate_xor_distance(mn.pastel_id,
int(chunk.chunk_id))
ChunkMnDistance.create(chunk=chunk,
masternode=mn,
distance=str(xor_distance))
def index_new_chunks():
"""
Select chunks which has not been indexed (XOR distance is not calculated) and calculate XOR distance for them.
"""
chunk_qs = Chunk.select().where(Chunk.indexed == False)
chunk_ids = [int(c.chunk_id) for c in chunk_qs]
if len(chunk_ids):
chunk_storage_logger.info(
'index_new_chunks found {} unprocessed chunks. Processing...'.
format(len(chunk_ids)))
calculate_xor_distances_for_chunks(chunk_ids)
# update all processed chunks with `indexed` = True
for chunk in chunk_qs:
chunk.indexed = True
Chunk.bulk_update(chunk_qs, fields=[Chunk.indexed])
chunk_storage_logger.info(
'Updating Chunk.indexed flag for processed chunks in DB')
# calculate chunk-mn-ranks as list of chunks was changed
recalculate_mn_chunk_ranking_table()
def calculate_xor_distances_for_chunks(chunk_ids):
"""
`chunk_ids` - list of chunks ids. Chunk ID is a very long integer.
"""
chunk_storage_logger.info(
'Calculating XOR distance for {} chunks...'.format(len(chunk_ids)))
chunk_ids_str = [str(x) for x in chunk_ids]
chunks_db = Chunk.select().where(Chunk.chunk_id.in_(chunk_ids_str))
counter = 0
for chunk in chunks_db:
for mn in Masternode.get_active_nodes():
# we store chunk.chunk_id as CharField, but essentially it's very long integer (more then 8 bytes,
# doesn't fit in database INT type)
xor_distance = calculate_xor_distance(mn.pastel_id,
int(chunk.chunk_id))
ChunkMnDistance.create(chunk=chunk,
masternode=mn,
distance=str(xor_distance))
counter += 1
chunk_storage_logger.info('..Caculated {} distances'.format(counter))
def receive_rpc_download_image(data, *args, **kwargs):
# fixme: image download logic should be much more complex.
# - masternode receive image download request, generate unique code and return to the client
# - MN has to validate that requestor pastelID is artwork owner pastelID
# - if check passed - collect required chunks from other MNs
# - assemble image and store it locally
# - provide interface for client to poll if image is ready or not.
image_hash = data['image_hash']
chunks_db = Chunk.select().where(Chunk.image_hash == image_hash)
if len(chunks_db) == 0:
return {"status": "ERROR", "mgs": "No chunks for given image"}
chunks = [
get_chunkmanager().get_chunk_data(int(x.chunk_id)) for x in chunks_db
]
try:
image_data = luby.decode(chunks)
except luby.NotEnoughChunks:
return {
"status": "ERROR",
"mgs": "Not enough chunks to reconstruct given image"
}
return {"status": "SUCCESS", "image_data": image_data}
def test_place_in_chunkstorage(self, chunkstorage):
image_data = png_1x1_data
regticket = get_regticket()
masternode_place_image_data_in_chunkstorage(regticket, image_data)
self.assertEqual(len(os.listdir('tmpstorage')), 2)
self.assertEqual(Chunk.select().count(), 2)
def test_xor_distance_task_without_chunks_without_masternodes(self):
Masternode.delete()
self.assertEqual(len(Chunk.select()), 0)
index_new_chunks()
self.assertEqual(len(ChunkMnDistance.select()), 0)
def test_xor_distance_task_without_chunks(self):
self.assertEqual(len(Chunk.select()), 0)
index_new_chunks()
self.assertEqual(len(ChunkMnDistance.select()), 0)
def test_no_act_ticket(self, get_blockchain_connection):
get_blockchain_connection().list_tickets.return_value = []
get_and_proccess_new_activation_tickets()
self.assertEqual(ActivationTicket.select().count(), 0)
self.assertEqual(Chunk.select().count(), 0)