def calculate_total_user_data(): os.makedirs(STORE_TOTAL_USER_DIRECTORY, exist_ok=True) max_time = datetime.utcnow() max_time = max_time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=pytz.UTC) stop_processing = False date_to_process = get_first_transaction_timestamp() date_last_processed = _get_last_processed_date() date_to_process = max(date_to_process, date_last_processed + timedelta(days=1)) log.debug('calculate: total user data') if date_to_process >= max_time: return state = _load_state(date_to_process) while not stop_processing: log.debug('analysing total user data for ' + date_to_process.strftime('%Y-%m-%d')) transactions = get_transaction_data( date_to_process, type_filter=['bank_MsgMultiSend', 'bank_MsgSend']) for transaction in transactions: # log.debug(transaction) type = transaction[0] block = transaction[1] timestamp = transaction[2] tx_hash = transaction[3] amount = int(transaction[4]) currency = transaction[5] from_address = transaction[6] to_address = transaction[7] tax_amount = transaction[8] tax_currency = transaction[9] if currency not in state.keys(): state[currency] = {} if from_address not in state[currency]: state[currency][from_address] = { 'first_seen_timestamp': timestamp, } state[currency][from_address]['last_seen_timestamp'] = timestamp _save_state(date_to_process, state) date_to_process += timedelta(days=1) if date_to_process >= max_time: stop_processing = True
def update_balances(token): symbol = token['symbol'] symbol_dir = BASE_DIRECTORY + symbol os.makedirs(symbol_dir, exist_ok=True) max_time = datetime.utcnow() max_time = max_time.replace(hour=0, minute=0, second=0, microsecond=0) stop_processing = False date_to_process = _get_next_time_to_process(symbol, symbol_dir) if date_to_process >= max_time: stop_processing = True state = _load_state(symbol_dir, date_to_process) log.debug('manage balances for ' + symbol) while not stop_processing: transactions = get_transaction_data(symbol, date_to_process) log.debug('managing balances for ' + str(date_to_process)) for transaction in transactions: block_number = transaction[0] timestamp = transaction[1] hash = transaction[2] nonce = transaction[3] block_hash = transaction[4] from_address = transaction[5] to_address = transaction[6] value = int(transaction[7]) token_decimal = transaction[8] transaction_index = transaction[9] gas = transaction[10] gas_price = transaction[11] gas_used = transaction[12] cumulative_gas_used = transaction[13] input = transaction[14] confirmations = transaction[15] if from_address in state.keys(): from_account = state[from_address] else: from_account = None if to_address in state.keys(): to_account = state[to_address] else: to_account = { 'balance': 0, 'balance_normalized': 0, } state[to_address] = to_account # change balances if from_account: from_account['balance'] -= value from_account['balance'] = max(from_account['balance'], 0) to_account['balance'] += value # change normalized balances if from_address not in token['lending_contracts'] and to_address not in token['lending_contracts']: if from_account: from_account['balance_normalized'] -= value from_account['balance_normalized'] = max(from_account['balance_normalized'], 0) to_account['balance_normalized'] += value else: # when the transaction contains a lending address, the transaction should be visible from the point of view of the lending address # for the user address, it should not be visible if from_account and from_address in token['lending_contracts']: from_account['balance_normalized'] -= value from_account['balance_normalized'] = max(from_account['balance_normalized'], 0) if to_address in token['lending_contracts']: to_account['balance_normalized'] += value # all transactions are processed, saving state to a file _save_state(symbol_dir, date_to_process, state) date_to_process = date_to_process + timedelta(days=1) if date_to_process >= max_time: stop_processing = True
def calculate_daily_payment_data(): os.makedirs(STORE_DAILY_PAYMENTS_DIRECTORY, exist_ok=True) max_time = datetime.utcnow() max_time = max_time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=pytz.UTC) stop_processing = False date_to_process = get_first_transaction_timestamp() date_last_processed = _get_last_processed_date() date_to_process = max(date_to_process, date_last_processed + timedelta(days=1)) log.debug('calculate: total amount of coins per coin') # TODO remove all lines from STORE_DAILY_PAYMENTS_DIRECTORY which are in the future from date_to_process # TODO remove all files from STORE_DAILY_ADDRESS_PAYMENTS_DIRECTORY which are in the future from date_to_process if date_to_process >= max_time: return # with open(symbol_file, 'a') as file: while not stop_processing: log.debug('analysing payment data for ' + date_to_process.strftime('%Y-%m-%d')) transactions = get_transaction_data(date_to_process, type_filter=['bank_MsgMultiSend', 'bank_MsgSend']) token = dict() for transaction in transactions: type = transaction[0] block = transaction[1] timestamp = transaction[2] tx_hash = transaction[3] currency = transaction[5] tax_amount = transaction[8] tax_currency = transaction[9] if currency not in token.keys(): token[currency] = { 'total_amount': 0, 'payment_count': 0, 'total_tax_amount': 0, 'active_users': dict(), } amount = int(transaction[4]) from_address = transaction[6] to_address = transaction[7] tax_amount = int(transaction[8]) tax_currency = transaction[9] token[currency]['payment_count'] += 1 token[currency]['total_amount'] += amount token[currency]['total_tax_amount'] += tax_amount if from_address not in token[currency]['active_users'].keys(): token[currency]['active_users'][from_address] = { 'total_amount': 0, 'payment_count': 0, } token[currency]['active_users'][from_address]['total_amount'] += amount token[currency]['active_users'][from_address]['payment_count'] += 1 for currency in token.keys(): with open(os.path.join(STORE_DAILY_PAYMENTS_DIRECTORY, currency + '.csv'), 'a') as file: tax_rate = token[currency]['total_tax_amount'] / token[currency]['total_amount'] file.write(','.join([date_to_process.strftime('%Y-%m-%d'), str(token[currency]['total_amount']), str(token[currency]['payment_count']), f"{tax_rate:.15f}", ]) + '\n') os.makedirs(os.path.join(STORE_DAILY_ADDRESS_PAYMENTS_DIRECTORY, currency), exist_ok=True) with open(os.path.join(STORE_DAILY_ADDRESS_PAYMENTS_DIRECTORY, currency, date_to_process.strftime('%Y-%m-%d') + '.csv'), 'a') as file: for address in token[currency]['active_users'].keys(): file.write(','.join([address, str(token[currency]['active_users'][address]['total_amount']), str(token[currency]['active_users'][address]['payment_count']), ]) + '\n') date_to_process += timedelta(days=1) if date_to_process >= max_time: stop_processing = True
def update_realized_market_capitalization(): os.makedirs(STORE_REALIZED_MARKET_CAP_DATA, exist_ok=True) max_time = datetime.utcnow() max_time = max_time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=pytz.UTC) stop_processing = False date_to_process = get_first_transaction_timestamp() date_last_processed = _get_last_processed_date() date_to_process = max(date_to_process, date_last_processed + timedelta(days=1)) if date_to_process >= max_time: return state = _load_state(date_to_process) log.debug('calculate: realized market cap') while not stop_processing: transactions = get_transaction_data( date_to_process, type_filter=['bank_MsgMultiSend', 'bank_MsgSend']) log.debug('processing realized market cap for: ' + str(date_to_process)) for transaction in transactions: start_date = datetime.now() type = transaction[0] block = transaction[1] timestamp = transaction[2] tx_hash = transaction[3] amount = int(transaction[4]) currency = transaction[5] from_address = transaction[6] to_address = transaction[7] tax_amount = int(transaction[8]) tax_currency = transaction[9] price = 0.1 # TODO get the correct market price per token per date # first_market_price_date = get_first_market_price_date(symbol) # # if not first_market_price_date: # log.debug("no market price available") # return # # if int(timestamp) < first_market_price_date.timestamp(): # # if init_price: # price = init_price # else: # price = 0 # else: # price = get_local_exchange_rate(symbol, datetime.utcfromtimestamp(int(timestamp))) if currency not in state.keys(): state[currency] = {} if from_address in state[currency].keys(): from_account = state[currency][from_address] else: from_account = None if to_address in state[currency].keys(): to_account = state[currency][to_address] else: to_account = { 'balance': 0, 'data': [], } state[currency][to_address] = to_account # # add transaction to the from-account # if from_account: remaining_value = amount while remaining_value > 0: try: from_amount = from_account['data'][0][1] except Exception: # log.debug(transaction) break if remaining_value < from_amount: from_account['data'][0][1] -= remaining_value remaining_value = 0 from_account['data'][0][2] = price else: remaining_value -= from_amount from_account['data'] = from_account['data'][1:] from_account['balance'] = max( 0, int(from_account['balance']) - amount) # # add transaction to the to-account # to_account['data'].append([timestamp, amount, price]) to_account['balance'] = int(to_account['balance']) + amount end_date = datetime.now() # print('calculation time: ' + str((end_date - start_date).total_seconds() * 1000)) # # all transactions are processed, saving state to a file # _save_state(date_to_process, state) date_to_process = date_to_process + timedelta(days=1) if date_to_process >= max_time: stop_processing = True
def update_realized_market_capitalization(token): symbol = token['symbol'] init_price = token.get('init_price') symbol_dir = BASE_DIRECTORY + symbol os.makedirs(symbol_dir, exist_ok=True) max_time = datetime.utcnow() max_time = max_time.replace(hour=0, minute=0, second=0, microsecond=0) stop_processing = False date_to_process = _get_next_time_to_process(symbol, symbol_dir) if date_to_process >= max_time: stop_processing = True state = _load_state(symbol_dir, date_to_process) log.debug('updating realized market cap for ' + symbol) while not stop_processing: transactions = get_transaction_data(symbol, date_to_process) log.debug('processing: ' + str(date_to_process)) for transaction in transactions: block_number = transaction[0] timestamp = transaction[1] hash = transaction[2] nonce = transaction[3] block_hash = transaction[4] from_address = transaction[5] to_address = transaction[6] value = int(transaction[7]) token_decimal = transaction[8] transaction_index = transaction[9] gas = transaction[10] gas_price = transaction[11] gas_used = transaction[12] cumulative_gas_used = transaction[13] input = transaction[14] confirmations = transaction[15] first_market_price_date = get_first_market_price_date(symbol) if not first_market_price_date: log.debug("no market price available") return if int(timestamp) < first_market_price_date.timestamp(): if init_price: price = init_price else: price = 0 else: price = get_local_exchange_rate( symbol, datetime.utcfromtimestamp(int(timestamp))) if from_address in state.keys(): from_account = state[from_address] else: from_account = None if to_address in state.keys(): to_account = state[to_address] else: to_account = { 'balance': 0, 'data': [], } state[to_address] = to_account # # add transaction to the from-account # if from_account and from_address != '0x0000000000000000000000000000000000000000': remaining_value = value while remaining_value > 0: try: from_amount = from_account['data'][0][1] except Exception: log.debug(transaction) break if remaining_value < from_amount: from_account['data'][0][1] -= remaining_value remaining_value = 0 from_account['data'][0][2] = price else: remaining_value -= from_amount from_account['data'] = from_account['data'][1:] from_balance = 0 for entry in from_account['data']: from_balance += int(entry[1]) from_account['balance'] = from_balance # # add transaction to the to-account # to_account['data'].append([timestamp, value, price]) to_balance = 0 for entry in to_account['data']: to_balance += int(entry[1]) to_account['balance'] = to_balance # all transactions are processed, saving state to a file _save_state(symbol_dir, date_to_process, state) date_to_process = date_to_process + timedelta(days=1) if date_to_process >= max_time: stop_processing = True
def calculate_daily_transaction_data(): # symbol = token['symbol'] # symbol_file = STORE_DIRECTORY + symbol os.makedirs(STORE_DAILY_TRANSACTIONS_DIRECTORY, exist_ok=True) max_time = datetime.utcnow() max_time = max_time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=pytz.UTC) stop_processing = False date_to_process = get_first_transaction_timestamp() date_last_processed = _get_last_processed_date() date_to_process = max(date_to_process, date_last_processed + timedelta(days=1)) log.debug('calculate: total amount of transactions per coin per type') # TODO remove all lines from STORE_DAILY_PAYMENTS_DIRECTORY which are in the future from date_to_process # TODO remove all files from STORE_DAILY_ADDRESS_PAYMENTS_DIRECTORY which are in the future from date_to_process if date_to_process >= max_time: return # with open(symbol_file, 'a') as file: while not stop_processing: log.debug('analysing transaction data for ' + date_to_process.strftime('%Y-%m-%d')) transactions = get_transaction_data(date_to_process) types = dict() for transaction in transactions: type = transaction[0] block = transaction[1] timestamp = transaction[2] tx_hash = transaction[3] if type not in types.keys(): types[type] = { 'count': 0, 'currencies': dict(), } currency = None if type == 'bank_MsgMultiSend': currency = transaction[5] elif type == 'bank_MsgSend': currency = transaction[5] elif type == 'distribution_MsgWithdrawDelegationReward': currency = None elif type == 'distribution_MsgWithdrawValidatorCommission': currency = None elif type == 'gov_MsgDeposit': currency = transaction[7] elif type == 'gov_MsgSubmitProposal': currency = None elif type == 'market_MsgSwap': currency = None elif type == 'oracle_MsgDelegateFeedConsent': currency = None elif type == 'oracle_MsgExchangeRatePrevote': currency = transaction[5] elif type == 'oracle_MsgExchangeRateVote': currency = transaction[5] elif type == 'staking_MsgCreateValidator': currency = transaction[6] elif type == 'staking_MsgDelegate': currency = transaction[7] elif type == 'staking_MsgEditValidator': currency = None if currency and currency not in types[type]['currencies']: types[type]['currencies'][currency] = { 'count': 0, } if currency: types[type]['currencies'][currency]['count'] += 1 else: types[type]['count'] += 1 # print(types) for type in types.keys(): os.makedirs(os.path.join(STORE_DAILY_TRANSACTIONS_DIRECTORY, type), exist_ok=True) if len(types[type]['currencies']) > 0: for currency in types[type]['currencies']: with open(os.path.join(STORE_DAILY_TRANSACTIONS_DIRECTORY, type, currency + '.csv'), 'a') as file: file.write(','.join([date_to_process.strftime('%Y-%m-%d'), str(types[type]['currencies'][currency]['count']), ]) + '\n') else: with open(os.path.join(STORE_DAILY_TRANSACTIONS_DIRECTORY, type, 'default.csv'), 'a') as file: file.write(','.join( [date_to_process.strftime('%Y-%m-%d'), str(types[type]['count']), ]) + '\n') date_to_process += timedelta(days=1) if date_to_process >= max_time: stop_processing = True