def upload_avro_files(self, block_number, all_block_data):
for key, value in all_block_data.items():
c = write_to_avro(value, key)
file_name = f"{key}/{block_number}.avro"
upload_file_to_gcs(self.avro_bucket, file_name, c)
def save_avro_files_locally(self, file_name, all_block_data):
for key, value in all_block_data.items():
c = write_to_avro(value, key)
final_file_name = f"{AVRO_LOCAL_STORAGE}/{key}/{file_name}.avro"
bytesio_to_file(c, final_file_name)
logger.info(f"Saved data for {file_name}")
table_names = {
'Activation': 'activation',
'BalanceUpdate': 'balance_update',
'Ballot': 'ballot',
'Block': 'block',
'Delegation': 'delegation',
'DoubleBakingEvidence': 'double_baking_evidence',
'DoubleEndorsementEvidence': 'double_endorsement_evidence',
'Endorsement': 'endorsement',
'Nonce': 'nonce',
'Operation': 'operation',
'Origination': 'origination',
'Proposal': 'proposal',
'Reveal': 'reveal',
'Transaction': 'transaction',
}
start_block_number = 0
end_block_number = 870000
batch_size = 1000
for data_type in table_names.keys():
for batch in range(start_block_number, end_block_number, batch_size):
final_file_name = f"{settings.BATCH_AVRO_LOCAL_STORAGE}/{data_type}/{data_type.lower()}_{batch}_{batch+batch_size-1}.avro"
data_list = []
for b in range(batch, batch + batch_size):
file_name = f"{settings.AVRO_LOCAL_STORAGE}/{data_type}/{b}.avro"
data_list += read_from_avro(file_name)
c = write_to_avro(data_list, data_type)
bytesio_to_file(c, final_file_name)
logger.info(f"Successfully wrote file {final_file_name}")
def test_transform_operation_types(block_number):
response = json.loads(get_response_from_mocks(block_number))
blocks = transform_block(response)
operation_types = transform_operation_types(blocks, response)
for key, value in operation_types.items():
c = write_to_avro(value, key)
def test_transform_operation(block_number):
response = json.loads(get_response_from_mocks(block_number))
blocks = transform_block(response)
operations = transform_operations(blocks, response)
c = write_to_avro(operations, "Operation")
def test_transform_balance_update(block_number):
response = json.loads(get_response_from_mocks(block_number))
blocks = transform_block(response)
balance_updates = transform_balance_updates(blocks, response)
c = write_to_avro(balance_updates, "BalanceUpdate")
def test_transform_block(block_number):
response = json.loads(get_response_from_mocks(block_number))
blocks = transform_block(response)
c = write_to_avro(blocks, "Block")