def calculate_total_user_data():
os.makedirs(STORE_TOTAL_USER_DIRECTORY, exist_ok=True)
max_time = datetime.utcnow()
max_time = max_time.replace(hour=0,
minute=0,
second=0,
microsecond=0,
tzinfo=pytz.UTC)
stop_processing = False
date_to_process = get_first_transaction_timestamp()
date_last_processed = _get_last_processed_date()
date_to_process = max(date_to_process,
date_last_processed + timedelta(days=1))
log.debug('calculate: total user data')
if date_to_process >= max_time:
return
state = _load_state(date_to_process)
while not stop_processing:
log.debug('analysing total user data for ' +
date_to_process.strftime('%Y-%m-%d'))
transactions = get_transaction_data(
date_to_process, type_filter=['bank_MsgMultiSend', 'bank_MsgSend'])
for transaction in transactions:
# log.debug(transaction)
type = transaction[0]
block = transaction[1]
timestamp = transaction[2]
tx_hash = transaction[3]
amount = int(transaction[4])
currency = transaction[5]
from_address = transaction[6]
to_address = transaction[7]
tax_amount = transaction[8]
tax_currency = transaction[9]
if currency not in state.keys():
state[currency] = {}
if from_address not in state[currency]:
state[currency][from_address] = {
'first_seen_timestamp': timestamp,
}
state[currency][from_address]['last_seen_timestamp'] = timestamp
_save_state(date_to_process, state)
date_to_process += timedelta(days=1)
if date_to_process >= max_time:
stop_processing = True
def update_balances(token):
symbol = token['symbol']
symbol_dir = BASE_DIRECTORY + symbol
os.makedirs(symbol_dir, exist_ok=True)
max_time = datetime.utcnow()
max_time = max_time.replace(hour=0, minute=0, second=0, microsecond=0)
stop_processing = False
date_to_process = _get_next_time_to_process(symbol, symbol_dir)
if date_to_process >= max_time:
stop_processing = True
state = _load_state(symbol_dir, date_to_process)
log.debug('manage balances for ' + symbol)
while not stop_processing:
transactions = get_transaction_data(symbol, date_to_process)
log.debug('managing balances for ' + str(date_to_process))
for transaction in transactions:
block_number = transaction[0]
timestamp = transaction[1]
hash = transaction[2]
nonce = transaction[3]
block_hash = transaction[4]
from_address = transaction[5]
to_address = transaction[6]
value = int(transaction[7])
token_decimal = transaction[8]
transaction_index = transaction[9]
gas = transaction[10]
gas_price = transaction[11]
gas_used = transaction[12]
cumulative_gas_used = transaction[13]
input = transaction[14]
confirmations = transaction[15]
if from_address in state.keys():
from_account = state[from_address]
else:
from_account = None
if to_address in state.keys():
to_account = state[to_address]
else:
to_account = {
'balance': 0,
'balance_normalized': 0,
}
state[to_address] = to_account
# change balances
if from_account:
from_account['balance'] -= value
from_account['balance'] = max(from_account['balance'], 0)
to_account['balance'] += value
# change normalized balances
if from_address not in token['lending_contracts'] and to_address not in token['lending_contracts']:
if from_account:
from_account['balance_normalized'] -= value
from_account['balance_normalized'] = max(from_account['balance_normalized'], 0)
to_account['balance_normalized'] += value
else:
# when the transaction contains a lending address, the transaction should be visible from the point of view of the lending address
# for the user address, it should not be visible
if from_account and from_address in token['lending_contracts']:
from_account['balance_normalized'] -= value
from_account['balance_normalized'] = max(from_account['balance_normalized'], 0)
if to_address in token['lending_contracts']:
to_account['balance_normalized'] += value
# all transactions are processed, saving state to a file
_save_state(symbol_dir, date_to_process, state)
date_to_process = date_to_process + timedelta(days=1)
if date_to_process >= max_time:
stop_processing = True
def calculate_daily_payment_data():
os.makedirs(STORE_DAILY_PAYMENTS_DIRECTORY, exist_ok=True)
max_time = datetime.utcnow()
max_time = max_time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=pytz.UTC)
stop_processing = False
date_to_process = get_first_transaction_timestamp()
date_last_processed = _get_last_processed_date()
date_to_process = max(date_to_process, date_last_processed + timedelta(days=1))
log.debug('calculate: total amount of coins per coin')
# TODO remove all lines from STORE_DAILY_PAYMENTS_DIRECTORY which are in the future from date_to_process
# TODO remove all files from STORE_DAILY_ADDRESS_PAYMENTS_DIRECTORY which are in the future from date_to_process
if date_to_process >= max_time:
return
# with open(symbol_file, 'a') as file:
while not stop_processing:
log.debug('analysing payment data for ' + date_to_process.strftime('%Y-%m-%d'))
transactions = get_transaction_data(date_to_process, type_filter=['bank_MsgMultiSend', 'bank_MsgSend'])
token = dict()
for transaction in transactions:
type = transaction[0]
block = transaction[1]
timestamp = transaction[2]
tx_hash = transaction[3]
currency = transaction[5]
tax_amount = transaction[8]
tax_currency = transaction[9]
if currency not in token.keys():
token[currency] = {
'total_amount': 0,
'payment_count': 0,
'total_tax_amount': 0,
'active_users': dict(),
}
amount = int(transaction[4])
from_address = transaction[6]
to_address = transaction[7]
tax_amount = int(transaction[8])
tax_currency = transaction[9]
token[currency]['payment_count'] += 1
token[currency]['total_amount'] += amount
token[currency]['total_tax_amount'] += tax_amount
if from_address not in token[currency]['active_users'].keys():
token[currency]['active_users'][from_address] = {
'total_amount': 0,
'payment_count': 0,
}
token[currency]['active_users'][from_address]['total_amount'] += amount
token[currency]['active_users'][from_address]['payment_count'] += 1
for currency in token.keys():
with open(os.path.join(STORE_DAILY_PAYMENTS_DIRECTORY, currency + '.csv'), 'a') as file:
tax_rate = token[currency]['total_tax_amount'] / token[currency]['total_amount']
file.write(','.join([date_to_process.strftime('%Y-%m-%d'),
str(token[currency]['total_amount']),
str(token[currency]['payment_count']),
f"{tax_rate:.15f}",
]) + '\n')
os.makedirs(os.path.join(STORE_DAILY_ADDRESS_PAYMENTS_DIRECTORY, currency), exist_ok=True)
with open(os.path.join(STORE_DAILY_ADDRESS_PAYMENTS_DIRECTORY,
currency,
date_to_process.strftime('%Y-%m-%d') + '.csv'), 'a') as file:
for address in token[currency]['active_users'].keys():
file.write(','.join([address,
str(token[currency]['active_users'][address]['total_amount']),
str(token[currency]['active_users'][address]['payment_count']),
]) + '\n')
date_to_process += timedelta(days=1)
if date_to_process >= max_time:
stop_processing = True
def update_realized_market_capitalization():
os.makedirs(STORE_REALIZED_MARKET_CAP_DATA, exist_ok=True)
max_time = datetime.utcnow()
max_time = max_time.replace(hour=0,
minute=0,
second=0,
microsecond=0,
tzinfo=pytz.UTC)
stop_processing = False
date_to_process = get_first_transaction_timestamp()
date_last_processed = _get_last_processed_date()
date_to_process = max(date_to_process,
date_last_processed + timedelta(days=1))
if date_to_process >= max_time:
return
state = _load_state(date_to_process)
log.debug('calculate: realized market cap')
while not stop_processing:
transactions = get_transaction_data(
date_to_process, type_filter=['bank_MsgMultiSend', 'bank_MsgSend'])
log.debug('processing realized market cap for: ' +
str(date_to_process))
for transaction in transactions:
start_date = datetime.now()
type = transaction[0]
block = transaction[1]
timestamp = transaction[2]
tx_hash = transaction[3]
amount = int(transaction[4])
currency = transaction[5]
from_address = transaction[6]
to_address = transaction[7]
tax_amount = int(transaction[8])
tax_currency = transaction[9]
price = 0.1
# TODO get the correct market price per token per date
# first_market_price_date = get_first_market_price_date(symbol)
#
# if not first_market_price_date:
# log.debug("no market price available")
# return
#
# if int(timestamp) < first_market_price_date.timestamp():
#
# if init_price:
# price = init_price
# else:
# price = 0
# else:
# price = get_local_exchange_rate(symbol, datetime.utcfromtimestamp(int(timestamp)))
if currency not in state.keys():
state[currency] = {}
if from_address in state[currency].keys():
from_account = state[currency][from_address]
else:
from_account = None
if to_address in state[currency].keys():
to_account = state[currency][to_address]
else:
to_account = {
'balance': 0,
'data': [],
}
state[currency][to_address] = to_account
#
# add transaction to the from-account
#
if from_account:
remaining_value = amount
while remaining_value > 0:
try:
from_amount = from_account['data'][0][1]
except Exception:
# log.debug(transaction)
break
if remaining_value < from_amount:
from_account['data'][0][1] -= remaining_value
remaining_value = 0
from_account['data'][0][2] = price
else:
remaining_value -= from_amount
from_account['data'] = from_account['data'][1:]
from_account['balance'] = max(
0,
int(from_account['balance']) - amount)
#
# add transaction to the to-account
#
to_account['data'].append([timestamp, amount, price])
to_account['balance'] = int(to_account['balance']) + amount
end_date = datetime.now()
# print('calculation time: ' + str((end_date - start_date).total_seconds() * 1000))
#
# all transactions are processed, saving state to a file
#
_save_state(date_to_process, state)
date_to_process = date_to_process + timedelta(days=1)
if date_to_process >= max_time:
stop_processing = True
def update_realized_market_capitalization(token):
symbol = token['symbol']
init_price = token.get('init_price')
symbol_dir = BASE_DIRECTORY + symbol
os.makedirs(symbol_dir, exist_ok=True)
max_time = datetime.utcnow()
max_time = max_time.replace(hour=0, minute=0, second=0, microsecond=0)
stop_processing = False
date_to_process = _get_next_time_to_process(symbol, symbol_dir)
if date_to_process >= max_time:
stop_processing = True
state = _load_state(symbol_dir, date_to_process)
log.debug('updating realized market cap for ' + symbol)
while not stop_processing:
transactions = get_transaction_data(symbol, date_to_process)
log.debug('processing: ' + str(date_to_process))
for transaction in transactions:
block_number = transaction[0]
timestamp = transaction[1]
hash = transaction[2]
nonce = transaction[3]
block_hash = transaction[4]
from_address = transaction[5]
to_address = transaction[6]
value = int(transaction[7])
token_decimal = transaction[8]
transaction_index = transaction[9]
gas = transaction[10]
gas_price = transaction[11]
gas_used = transaction[12]
cumulative_gas_used = transaction[13]
input = transaction[14]
confirmations = transaction[15]
first_market_price_date = get_first_market_price_date(symbol)
if not first_market_price_date:
log.debug("no market price available")
return
if int(timestamp) < first_market_price_date.timestamp():
if init_price:
price = init_price
else:
price = 0
else:
price = get_local_exchange_rate(
symbol, datetime.utcfromtimestamp(int(timestamp)))
if from_address in state.keys():
from_account = state[from_address]
else:
from_account = None
if to_address in state.keys():
to_account = state[to_address]
else:
to_account = {
'balance': 0,
'data': [],
}
state[to_address] = to_account
#
# add transaction to the from-account
#
if from_account and from_address != '0x0000000000000000000000000000000000000000':
remaining_value = value
while remaining_value > 0:
try:
from_amount = from_account['data'][0][1]
except Exception:
log.debug(transaction)
break
if remaining_value < from_amount:
from_account['data'][0][1] -= remaining_value
remaining_value = 0
from_account['data'][0][2] = price
else:
remaining_value -= from_amount
from_account['data'] = from_account['data'][1:]
from_balance = 0
for entry in from_account['data']:
from_balance += int(entry[1])
from_account['balance'] = from_balance
#
# add transaction to the to-account
#
to_account['data'].append([timestamp, value, price])
to_balance = 0
for entry in to_account['data']:
to_balance += int(entry[1])
to_account['balance'] = to_balance
# all transactions are processed, saving state to a file
_save_state(symbol_dir, date_to_process, state)
date_to_process = date_to_process + timedelta(days=1)
if date_to_process >= max_time:
stop_processing = True
def calculate_daily_transaction_data():
# symbol = token['symbol']
# symbol_file = STORE_DIRECTORY + symbol
os.makedirs(STORE_DAILY_TRANSACTIONS_DIRECTORY, exist_ok=True)
max_time = datetime.utcnow()
max_time = max_time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=pytz.UTC)
stop_processing = False
date_to_process = get_first_transaction_timestamp()
date_last_processed = _get_last_processed_date()
date_to_process = max(date_to_process, date_last_processed + timedelta(days=1))
log.debug('calculate: total amount of transactions per coin per type')
# TODO remove all lines from STORE_DAILY_PAYMENTS_DIRECTORY which are in the future from date_to_process
# TODO remove all files from STORE_DAILY_ADDRESS_PAYMENTS_DIRECTORY which are in the future from date_to_process
if date_to_process >= max_time:
return
# with open(symbol_file, 'a') as file:
while not stop_processing:
log.debug('analysing transaction data for ' + date_to_process.strftime('%Y-%m-%d'))
transactions = get_transaction_data(date_to_process)
types = dict()
for transaction in transactions:
type = transaction[0]
block = transaction[1]
timestamp = transaction[2]
tx_hash = transaction[3]
if type not in types.keys():
types[type] = {
'count': 0,
'currencies': dict(),
}
currency = None
if type == 'bank_MsgMultiSend':
currency = transaction[5]
elif type == 'bank_MsgSend':
currency = transaction[5]
elif type == 'distribution_MsgWithdrawDelegationReward':
currency = None
elif type == 'distribution_MsgWithdrawValidatorCommission':
currency = None
elif type == 'gov_MsgDeposit':
currency = transaction[7]
elif type == 'gov_MsgSubmitProposal':
currency = None
elif type == 'market_MsgSwap':
currency = None
elif type == 'oracle_MsgDelegateFeedConsent':
currency = None
elif type == 'oracle_MsgExchangeRatePrevote':
currency = transaction[5]
elif type == 'oracle_MsgExchangeRateVote':
currency = transaction[5]
elif type == 'staking_MsgCreateValidator':
currency = transaction[6]
elif type == 'staking_MsgDelegate':
currency = transaction[7]
elif type == 'staking_MsgEditValidator':
currency = None
if currency and currency not in types[type]['currencies']:
types[type]['currencies'][currency] = {
'count': 0,
}
if currency:
types[type]['currencies'][currency]['count'] += 1
else:
types[type]['count'] += 1
# print(types)
for type in types.keys():
os.makedirs(os.path.join(STORE_DAILY_TRANSACTIONS_DIRECTORY, type), exist_ok=True)
if len(types[type]['currencies']) > 0:
for currency in types[type]['currencies']:
with open(os.path.join(STORE_DAILY_TRANSACTIONS_DIRECTORY, type, currency + '.csv'), 'a') as file:
file.write(','.join([date_to_process.strftime('%Y-%m-%d'),
str(types[type]['currencies'][currency]['count']),
]) + '\n')
else:
with open(os.path.join(STORE_DAILY_TRANSACTIONS_DIRECTORY, type, 'default.csv'), 'a') as file:
file.write(','.join(
[date_to_process.strftime('%Y-%m-%d'),
str(types[type]['count']),
]) + '\n')
date_to_process += timedelta(days=1)
if date_to_process >= max_time:
stop_processing = True