def _deserialize_transaction(grant_id: int, rawtx: Dict[str,
Any]) -> LedgerAction:
"""May raise:
- DeserializationError
- KeyError
- UnknownAsset
"""
timestamp = deserialize_timestamp_from_date(
date=rawtx['timestamp'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin API',
skip_milliseconds=True,
)
asset = get_gitcoin_asset(symbol=rawtx['asset'],
token_address=rawtx['token_address'])
raw_amount = deserialize_int_from_str(symbol=rawtx['amount'],
location='gitcoin api')
amount = asset_normalized_value(raw_amount, asset)
if amount == ZERO:
raise ZeroGitcoinAmount()
# let's use gitcoin's calculated rate for now since they include it in the response
usd_value = Price(
ZERO) if rawtx['usd_value'] is None else deserialize_price(
rawtx['usd_value']) # noqa: E501
rate = Price(ZERO) if usd_value == ZERO else Price(usd_value / amount)
raw_txid = rawtx['tx_hash']
tx_type, tx_id = process_gitcoin_txid(key='tx_hash', entry=rawtx)
# until we figure out if we can use it https://github.com/gitcoinco/web/issues/9255#issuecomment-874537144 # noqa: E501
clr_round = _calculate_clr_round(timestamp, rawtx)
notes = f'Gitcoin grant {grant_id} event' if not clr_round else f'Gitcoin grant {grant_id} event in clr_round {clr_round}' # noqa: E501
return LedgerAction(
identifier=1, # whatever -- does not end up in the DB
timestamp=timestamp,
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=AssetAmount(amount),
asset=asset,
rate=rate,
rate_asset=A_USD,
link=raw_txid,
notes=notes,
extra_data=GitcoinEventData(
tx_id=tx_id,
grant_id=grant_id,
clr_round=clr_round,
tx_type=tx_type,
),
)
def trade_from_coinbase(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns a coinbase transaction into a rotkehlchen Trade.
https://developers.coinbase.com/api/v2?python#buys
If the coinbase transaction is not a trade related transaction returns None
Throws:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
if raw_trade['status'] != 'completed':
# We only want to deal with completed trades
return None
if raw_trade['instant']:
raw_time = raw_trade['created_at']
else:
raw_time = raw_trade['payout_at']
timestamp = deserialize_timestamp_from_date(raw_time, 'iso8601',
'coinbase')
trade_type = deserialize_trade_type(raw_trade['resource'])
tx_amount = deserialize_asset_amount(raw_trade['amount']['amount'])
tx_asset = asset_from_coinbase(raw_trade['amount']['currency'],
time=timestamp)
native_amount = deserialize_asset_amount(raw_trade['subtotal']['amount'])
native_asset = asset_from_coinbase(raw_trade['subtotal']['currency'],
time=timestamp)
# in coinbase you are buying/selling tx_asset for native_asset
pair = TradePair(f'{tx_asset.identifier}_{native_asset.identifier}')
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee']['amount'])
fee_asset = asset_from_coinbase(raw_trade['fee']['currency'],
time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['id']),
)
def _read_asset_movements(self, filepath: str) -> List[AssetMovement]:
"""Reads a csv account type report and extracts the AssetMovements"""
with open(filepath, newline='') as csvfile:
reader = csv.DictReader(csvfile)
movements = []
for row in reader:
try:
if row['type'] in ('withdrawal', 'deposit'):
timestamp = deserialize_timestamp_from_date(
row['time'],
'iso8601',
'coinbasepro',
)
asset = Asset(row['amount/balance unit'])
movements.append(AssetMovement(
location=Location.COINBASEPRO,
category=deserialize_asset_movement_category(row['type']),
address=None, # can't get it from csv data
transaction_id=None, # can't get it from csv data
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(row['amount']),
fee_asset=asset,
# I don't see any fee in deposit withdrawals in coinbasepro
fee=Fee(ZERO),
link=str(row['transfer id']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring its deposit/withdrawal.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a Coinbasepro deposit/withdrawal. '
'Check logs for details. Ignoring it.',
)
log.error(
'Error processing a coinbasepro deposit/withdrawal.',
raw_asset_movement=row,
error=msg,
)
continue
return movements
def _consume_blockfi_trade(self, csv_row: Dict[str, Any]) -> None:
"""
Consume the file containing only trades from BlockFi. As per my investigations
(@yabirgb) this file can only contain confirmed trades.
- UnknownAsset
- DeserializationError
"""
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%Y-%m-%d %H:%M:%S',
location='BlockFi',
)
buy_asset = symbol_to_asset_or_token(csv_row['Buy Currency'])
buy_amount = deserialize_asset_amount(csv_row['Buy Quantity'])
sold_asset = symbol_to_asset_or_token(csv_row['Sold Currency'])
sold_amount = deserialize_asset_amount(csv_row['Sold Quantity'])
if sold_amount == ZERO:
log.debug(
f'Ignoring BlockFi trade with sold_amount equal to zero. {csv_row}'
)
return
rate = Price(buy_amount / sold_amount)
trade = Trade(
timestamp=timestamp,
location=Location.BLOCKFI,
base_asset=buy_asset,
quote_asset=sold_asset,
trade_type=TradeType.BUY,
amount=buy_amount,
rate=rate,
fee=None, # BlockFI doesn't provide this information
fee_currency=None,
link='',
notes=csv_row['Type'],
)
self.db.add_trades([trade])
def trade_from_ftx(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns an FTX transaction into a rotki Trade.
May raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected key
"""
# In the case of perpetuals and futures this fields can be None
if raw_trade.get('baseCurrency', None) is None:
return None
if raw_trade.get('quoteCurrency', None) is None:
return None
timestamp = deserialize_timestamp_from_date(raw_trade['time'], 'iso8601',
'FTX')
trade_type = TradeType.deserialize(raw_trade['side'])
base_asset = asset_from_ftx(raw_trade['baseCurrency'])
quote_asset = asset_from_ftx(raw_trade['quoteCurrency'])
amount = deserialize_asset_amount(raw_trade['size'])
rate = deserialize_price(raw_trade['price'])
fee_currency = asset_from_ftx(raw_trade['feeCurrency'])
fee = deserialize_fee(raw_trade['fee'])
return Trade(
timestamp=timestamp,
location=Location.FTX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(raw_trade['id']),
)
def _consume_blockfi_entry(self, csv_row: Dict[str, Any]) -> None:
"""
Process entry for BlockFi transaction history. Trades for this file are ignored
and istead should be extracted from the file containing only trades.
This method can raise:
- UnsupportedBlockFiEntry
- UnknownAsset
- DeserializationError
"""
if len(csv_row['Confirmed At']) != 0:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Confirmed At'],
formatstr='%Y-%m-%d %H:%M:%S',
location='BlockFi',
)
else:
log.debug(f'Ignoring unconfirmed BlockFi entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Cryptocurrency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Transaction Type']
# BlockFI doesn't provide information about fees
fee = Fee(ZERO)
fee_asset = A_USD # Can be whatever
if entry_type in ('Deposit', 'Wire Deposit', 'ACH Deposit'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'Wire Withdrawal', 'ACH Withdrawal'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest Payment', 'Bonus Payment',
'Referral Bonus'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type == 'Trade':
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def _import_cryptocom_associated_entries(self, data: Any,
tx_kind: str) -> None:
"""Look for events that have associated entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known kind: 'dynamic_coin_swap' or 'dust_conversion'
May raise:
- UnknownAsset if an unknown asset is encountered in the imported files
- KeyError if a row contains unexpected data entries
"""
multiple_rows: Dict[Any, Dict[str, Any]] = {}
investments_deposits: Dict[str, List[Any]] = defaultdict(list)
investments_withdrawals: Dict[str, List[Any]] = defaultdict(list)
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{tx_kind}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
if 'debited' not in multiple_rows[timestamp]:
multiple_rows[timestamp]['debited'] = []
multiple_rows[timestamp]['debited'].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_credited':
# They only is one credited row
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
multiple_rows[timestamp]['credited'] = row
elif row['Transaction Kind'] == f'{tx_kind}_deposit':
asset = row['Currency']
investments_deposits[asset].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_withdrawal':
asset = row['Currency']
investments_withdrawals[asset].append(row)
for timestamp in multiple_rows:
# When we convert multiple assets dust to CRO
# in one time, it will create multiple debited rows with
# the same timestamp
debited_rows = multiple_rows[timestamp]['debited']
credited_row = multiple_rows[timestamp]['credited']
total_debited_usd = functools.reduce(
lambda acc, row: acc + deserialize_asset_amount(row[
'Native Amount (in USD)']),
debited_rows,
ZERO,
)
# If the value of the transaction is too small (< 0,01$),
# crypto.com will display 0 as native amount
# if we have multiple debited rows, we can't import them
# since we can't compute their dedicated rates, so we skip them
if len(debited_rows) > 1 and total_debited_usd == 0:
return
if credited_row is not None and len(debited_rows) != 0:
for debited_row in debited_rows:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = symbol_to_asset_or_token(
credited_row['Currency'])
quote_asset = symbol_to_asset_or_token(
debited_row['Currency'])
part_of_total = (
FVal(1) if len(debited_rows) == 1 else
deserialize_asset_amount(
debited_row["Native Amount (in USD)"], ) /
total_debited_usd)
quote_amount_sold = deserialize_asset_amount(
debited_row['Amount'], ) * part_of_total
base_amount_bought = deserialize_asset_amount(
credited_row['Amount'], ) * part_of_total
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.BUY,
amount=AssetAmount(base_amount_bought),
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
# Compute investments profit
if len(investments_withdrawals) != 0:
for asset in investments_withdrawals:
asset_object = symbol_to_asset_or_token(asset)
if asset not in investments_deposits:
log.error(
f'Investment withdrawal without deposit at crypto.com. Ignoring '
f'staking info for asset {asset_object}', )
continue
# Sort by date in ascending order
withdrawals_rows = sorted(
investments_withdrawals[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
investments_rows = sorted(
investments_deposits[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
last_date = Timestamp(0)
for withdrawal in withdrawals_rows:
withdrawal_date = deserialize_timestamp_from_date(
date=withdrawal['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
amount_deposited = ZERO
for deposit in investments_rows:
deposit_date = deserialize_timestamp_from_date(
date=deposit['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if last_date < deposit_date <= withdrawal_date:
# Amount is negative
amount_deposited += deserialize_asset_amount(
deposit['Amount'])
amount_withdrawal = deserialize_asset_amount(
withdrawal['Amount'])
# Compute profit
profit = amount_withdrawal + amount_deposited
if profit >= ZERO:
last_date = withdrawal_date
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=withdrawal_date,
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(profit),
asset=asset_object,
rate=None,
rate_asset=None,
link=None,
notes=f'Stake profit for asset {asset}',
)
self.db_ledger.add_ledger_action(action)
def _import_cryptocom_double_entries(self, data: Any,
double_type: str) -> None:
"""Look for events that have double entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known double_type: 'dynamic_coin_swap' or 'dust_conversion'
"""
double_rows: Dict[Any, Dict[str, Any]] = {}
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{double_type}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in double_rows:
double_rows[timestamp] = {}
double_rows[timestamp]['debited'] = row
elif row['Transaction Kind'] == f'{double_type}_credited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in double_rows:
double_rows[timestamp] = {}
double_rows[timestamp]['credited'] = row
for timestamp in double_rows:
credited_row = double_rows[timestamp]['credited']
debited_row = double_rows[timestamp]['debited']
if credited_row is not None and debited_row is not None:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = Asset(credited_row['Currency'])
quote_asset = Asset(debited_row['Currency'])
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(
credited_row['Amount'])
quote_amount_sold = deserialize_asset_amount(
debited_row['Amount'])
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=TradeType.BUY,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
def _deserialize_ledger_action(
self, raw_data: Dict[str, Any]) -> Optional[LedgerAction]:
"""Processes a single transaction from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data.get('status', '') != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
get_key_if_has_val(raw_data, 'created_at'),
'iso8601',
'coinbase',
)
if 'type' in raw_data:
# The parent method filtered with 'from' attribute, so it is from another user.
# https://developers.coinbase.com/api/v2?python#transaction-resource
action_type = LedgerActionType.INCOME
if raw_data.get('type',
'') not in ('send', 'inflation_reward'):
msg = ('Non "send" or "inflation_reward" type '
'found in coinbase transactions processing')
raise DeserializationError(msg)
amount_data = raw_data.get('amount', {})
amount = deserialize_asset_amount(amount_data['amount'])
asset = asset_from_coinbase(amount_data['currency'],
time=timestamp)
native_amount_data = raw_data.get('native_amount', {})
native_amount = deserialize_asset_amount(
native_amount_data['amount'])
native_asset = asset_from_coinbase(
native_amount_data['currency'])
rate = ZERO
if amount_data and native_amount_data and native_amount and amount != ZERO:
rate = native_amount / amount
if 'details' in raw_data and 'title' in raw_data['details'] \
and 'subtitle' in raw_data['details'] and 'header' in raw_data['details']:
details = raw_data.get('details', {})
notes = (f"{details.get('title', '')} "
f"{details.get('subtitle', '')} "
f"{details.get('header', '')}")
else:
notes = ''
return LedgerAction(identifier=0,
location=Location.COINBASE,
action_type=action_type,
timestamp=timestamp,
asset=asset,
amount=amount,
rate=Price(rate),
rate_asset=native_asset,
link=str(raw_data['id']),
notes=notes)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase transaction with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase transaction with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'ledger action. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'ledger action {raw_data}. Error was: {msg}', )
return None
def _deserialize_asset_movement(
self,
raw_data: Dict[str, Any],
movement_type: AssetMovementCategory,
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from FTX and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] not in ('complete', 'confirmed'):
return None
timestamp = deserialize_timestamp_from_date(
raw_data['time'], 'iso8601', 'FTX')
amount = deserialize_asset_amount_force_positive(raw_data['size'])
asset = asset_from_ftx(raw_data['coin'])
fee = Fee(ZERO)
movement_category = movement_type
if raw_data.get('fee', None) is not None:
fee = deserialize_fee(raw_data['fee'])
address = raw_data.get('address', None)
if isinstance(address, dict):
address = raw_data['address'].get('address', None)
transaction_id = raw_data.get('txid', None)
return AssetMovement(
location=Location.FTX,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found FTX deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found FTX deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of an FTX '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
'Error processing FTX trade',
trade=raw_data,
error=msg,
)
return None
def _api_query(
self,
endpoint: str,
start_time: Optional[Timestamp] = None,
end_time: Optional[Timestamp] = None,
limit: int = PAGINATION_LIMIT,
paginate: bool = True,
) -> Union[List[Dict[str, Any]], Dict[str, List[Any]], Dict[str, Any]]:
"""Query FTX endpoint and retrieve all available information if pagination
is requested. In case of paginate being set to False only one request is made.
Can raise:
- RemoteError
"""
if not paginate:
final_data_no_pag = self._make_request(
endpoint=endpoint,
limit=limit,
start_time=start_time,
end_time=end_time,
)
return final_data_no_pag
# If there is pagination we follow the example from the official ftx python example
# https://github.com/ftexchange/ftx/blob/master/rest/client.py#L163
# In this case the strategy is a while loop leaving fixed the start_time (lower bound)
# and decreasing end time (the upper bound) until we fetch all the available information
new_end_time = end_time
ids = set()
final_data: List[Dict[str, Any]] = []
while True:
step = self._make_request(
endpoint=endpoint,
limit=limit,
start_time=start_time,
end_time=new_end_time,
)
if not isinstance(step, list):
raise RemoteError(
f'FTX pagination returned something different to a list for route {endpoint} '
f'with start_time {start_time} and end_time {end_time}. Result was {step}.',
)
# remove possible duplicates
deduped = [
r for r in step if 'id' in r.keys() and r['id'] not in ids
]
ids |= {r['id'] for r in deduped}
final_data.extend(deduped)
if len(step) == 0:
break
# Avoid deserialization error if there is a bad date
times = []
for t in step:
try:
times.append(
deserialize_timestamp_from_date(
t['time'], 'iso8601', 'ftx'))
except (DeserializationError, KeyError):
continue
if len(times) != 0:
new_end_time = min(times)
else:
self.msg_aggregator.add_error(
f'Error processing FTX trade history. Query step '
f'returned invalid entries when trying pagination for endpoint '
f'{endpoint} with start_time {start_time}, end_time {end_time} '
f'and page limit of {limit}.', )
break
if len(step) < limit:
break
return final_data
def retrieve_location_data(data_dir: Path) -> Optional[GeolocationData]:
"""This functions tries to get the country of the user based on the ip.
To do that it makes use of an open ip to country database and tries to obtain
the ip ussing UPnP protocol.
If UpNP fails we just return None
**IMPORTANT:** The ip never leaves the user's machine. It's all calculated locally.
"""
geoip_dir = data_dir / 'misc'
geoip_dir.mkdir(parents=True, exist_ok=True)
# get latest database version
metadata_query_failed = False
try:
response = requests.get(
url='https://api.github.com/repos/geoacumen/geoacumen-country/branches/master',
timeout=DEFAULT_TIMEOUT_TUPLE,
)
data = response.json()
date = deserialize_timestamp_from_date(
date=data['commit']['commit']['author']['date'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Analytics',
)
except requests.exceptions.RequestException as e:
log.debug(f'Failed to get metadata information for geoip file. {str(e)}')
metadata_query_failed = True
except (DeserializationError, JSONDecodeError) as e:
log.debug(f'Failed to deserialize date in metadata information for geoip file. {str(e)}')
metadata_query_failed = True
except KeyError as e:
log.debug(f'Github response for geoip file had missing key {str(e)}')
metadata_query_failed = True
if metadata_query_failed:
old_files = list(geoip_dir.glob('*.mmdb'))
if len(old_files) == 0:
return None
filename = old_files[0]
else:
filename = geoip_dir / f'geoip-{date}.mmdb'
if not filename.is_file():
# Remove old files
files = geoip_dir.glob('*.*')
for f in files:
f.unlink()
# Download latest version
try:
response = requests.get(
url='https://github.com/geoacumen/geoacumen-country/raw/master/Geoacumen-Country.mmdb', # noqa: E501
timeout=DEFAULT_TIMEOUT_TUPLE,
stream=True,
)
except requests.exceptions.RequestException as e:
log.debug(f'Failed to download geoip database file. {str(e)}')
return None
with open(filename, 'wb') as outfile:
response.raw.decode_content = True
shutil.copyfileobj(response.raw, outfile)
# get user ip
try:
u = miniupnpc.UPnP()
u.discoverdelay = 200
u.discover()
u.selectigd()
user_ip = u.externalipaddress()
except Exception as e: # pylint: disable=broad-except
# can raise base exception so we catch it
log.debug(f'Failed to get ip via UPnP for analytics. {str(e)}')
return None
try:
with maxminddb.open_database(filename) as reader:
data = reader.get(user_ip)
location = data['country']['iso_code']
if location == 'None':
return None
return GeolocationData(country_code=location)
except maxminddb.errors.InvalidDatabaseError as e:
filename.unlink()
log.debug(f'Failed to read database {str(e)}')
except ValueError as e:
log.debug(f'Wrong ip search {str(e)}')
return None
def _consume_grant_entry(self, entry: Dict[str,
Any]) -> Optional[LedgerAction]:
"""
Consumes a grant entry from the CSV and turns it into a LedgerAction
May raise:
- DeserializationError
- KeyError
- UnknownAsset
"""
if entry['Type'] != 'grant':
return None
timestamp = deserialize_timestamp_from_date(
date=entry['date'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin CSV',
skip_milliseconds=True,
)
usd_value = deserialize_asset_amount(entry['Value In USD'])
asset = get_asset_by_symbol(entry['token_name'])
if asset is None:
raise UnknownAsset(entry['token_name'])
token_amount = deserialize_asset_amount(entry['token_value'])
if token_amount == ZERO: # try to make up for https://github.com/gitcoinco/web/issues/9213
price = query_usd_price_zero_if_error(
asset=asset,
time=timestamp,
location=f'Gitcoin CSV entry {entry["txid"]}',
msg_aggregator=self.db.msg_aggregator,
)
if price == ZERO:
self.db.msg_aggregator.add_warning(
f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} '
f'due to amount being zero and inability to find price. Skipping.',
)
return None
# calculate the amount from price and value
token_amount = usd_value / price # type: ignore
match = self.grantid_re.search(entry['url'])
if match is None:
self.db.msg_aggregator.add_warning(
f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} '
f'due to inability to read grant id. Skipping.', )
return None
grant_id = int(match.group(1))
rate = Price(usd_value / token_amount)
raw_txid = entry['txid']
tx_type, tx_id = process_gitcoin_txid(key='txid', entry=entry)
return LedgerAction(
identifier=1, # whatever does not go in the DB
timestamp=timestamp,
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=token_amount,
asset=asset,
rate=rate,
rate_asset=A_USD, # let's use the rate gitcoin calculated
link=raw_txid,
notes=f'Gitcoin grant {grant_id} event',
extra_data=GitcoinEventData(
tx_id=tx_id,
grant_id=grant_id,
clr_round=None, # can't get round from CSV
tx_type=tx_type,
),
)
def _import_cryptocom_associated_entries(self, data: Any,
tx_kind: str) -> None:
"""Look for events that have associated entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known kind: 'dynamic_coin_swap' or 'dust_conversion'
"""
multiple_rows: Dict[Any, Dict[str, Any]] = {}
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{tx_kind}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
if 'debited' not in multiple_rows[timestamp]:
multiple_rows[timestamp]['debited'] = []
multiple_rows[timestamp]['debited'].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_credited':
# They only is one credited row
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
multiple_rows[timestamp]['credited'] = row
for timestamp in multiple_rows:
# When we convert multiple assets dust to CRO
# in one time, it will create multiple debited rows with
# the same timestamp
debited_rows = multiple_rows[timestamp]['debited']
credited_row = multiple_rows[timestamp]['credited']
total_debited_usd = functools.reduce(
lambda acc, row: acc + deserialize_asset_amount(row[
'Native Amount (in USD)']),
debited_rows,
FVal('0'),
)
# If the value of the transaction is too small (< 0,01$),
# crypto.com will display 0 as native amount
# if we have multiple debited rows, we can't import them
# since we can't compute their dedicated rates, so we skip them
if len(debited_rows) > 1 and total_debited_usd == 0:
return
if credited_row is not None and len(debited_rows) != 0:
for debited_row in debited_rows:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = Asset(credited_row['Currency'])
quote_asset = Asset(debited_row['Currency'])
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
part_of_total = (
FVal(1) if len(debited_rows) == 1 else
deserialize_asset_amount(
debited_row["Native Amount (in USD)"], ) /
total_debited_usd)
quote_amount_sold = deserialize_asset_amount(
debited_row['Amount'], ) * part_of_total
base_amount_bought = deserialize_asset_amount(
credited_row['Amount'], ) * part_of_total
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=TradeType.BUY,
amount=AssetAmount(base_amount_bought),
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
def _consume_nexo(self, csv_row: Dict[str, Any]) -> None:
"""
Consume CSV file from NEXO.
This method can raise:
- UnsupportedNexoEntry
- UnknownAsset
- DeserializationError
"""
ignored_entries = ('ExchangeToWithdraw', 'DepositToExchange')
if 'rejected' not in csv_row['Details']:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date / Time'],
formatstr='%Y-%m-%d %H:%M',
location='NEXO',
)
else:
log.debug(f'Ignoring rejected nexo entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Type']
transaction = csv_row['Transaction']
if entry_type in ('Deposit', 'ExchangeDepositedOn',
'LockingTermDeposit'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'WithdrawExchanged'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest', 'Bonus', 'Dividend'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=transaction,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ignored_entries:
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def trade_from_conversion(trade_a: Dict[str, Any],
trade_b: Dict[str, Any]) -> Optional[Trade]:
"""Turn information from a conversion into a trade
Mary raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
# Check that the status is complete
if trade_a['status'] != 'completed':
return None
# Trade b will represent the asset we are converting to
if trade_b['amount']['amount'].startswith('-'):
trade_a, trade_b = trade_b, trade_a
timestamp = deserialize_timestamp_from_date(trade_a['updated_at'],
'iso8601', 'coinbase')
tx_amount = AssetAmount(
abs(deserialize_asset_amount(trade_a['amount']['amount'])))
tx_asset = asset_from_coinbase(trade_a['amount']['currency'],
time=timestamp)
native_amount = deserialize_asset_amount(trade_b['amount']['amount'])
native_asset = asset_from_coinbase(trade_b['amount']['currency'],
time=timestamp)
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
# Obtain fee amount in the native currency using data from both trades
amount_after_fee = deserialize_asset_amount(
trade_b['native_amount']['amount'])
amount_before_fee = deserialize_asset_amount(
trade_a['native_amount']['amount'])
# amount_after_fee + amount_before_fee is a negative amount and the fee needs to be positive
conversion_native_fee_amount = abs(amount_after_fee + amount_before_fee)
if ZERO not in (tx_amount, conversion_native_fee_amount, amount_before_fee,
amount_after_fee):
# To get the asset in which the fee is nominated we pay attention to the creation
# date of each event. As per our hypothesis the fee is nominated in the asset
# for which the first transaction part was initialized
time_created_a = deserialize_timestamp_from_date(
date=trade_a['created_at'],
formatstr='iso8601',
location='coinbase',
)
time_created_b = deserialize_timestamp_from_date(
date=trade_b['created_at'],
formatstr='iso8601',
location='coinbase',
)
if time_created_a < time_created_b:
# We have the fee amount in the native currency. To get it in the
# converted asset we have to get the rate
asset_native_rate = tx_amount / abs(amount_before_fee)
fee_amount = Fee(conversion_native_fee_amount * asset_native_rate)
fee_asset = asset_from_coinbase(trade_a['amount']['currency'],
time=timestamp)
else:
trade_b_amount = abs(
deserialize_asset_amount(trade_b['amount']['amount']))
asset_native_rate = trade_b_amount / abs(amount_after_fee)
fee_amount = Fee(conversion_native_fee_amount * asset_native_rate)
fee_asset = asset_from_coinbase(trade_b['amount']['currency'],
time=timestamp)
else:
fee_amount = Fee(ZERO)
fee_asset = asset_from_coinbase(trade_a['amount']['currency'],
time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
# in coinbase you are buying/selling tx_asset for native_asset
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=TradeType.SELL,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(trade_a['trade']['id']),
)
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info for now (Aug 2020) on crypto.com, so we put 0 fees
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'crypto_earn_interest_paid',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = Asset(to_currency)
quote_asset = Asset(currency)
if quote_asset is None:
raise DeserializationError(
'Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = Asset(currency)
quote_asset = Asset(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in (
'crypto_earn_program_created',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dynamic_coin_swap_bonus_exchange_deposit',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCryptocomEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry', )
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
raw_data['created_at'],
'iso8601',
'coinbase',
)
# Only get address/transaction id for "send" type of transactions
address = None
transaction_id = None
# movement_category: Union[Literal['deposit'], Literal['withdrawal']]
if 'type' in raw_data:
# Then this should be a "send" which is the way Coinbase uses to send
# crypto outside of the exchange
# https://developers.coinbase.com/api/v2?python#transaction-resource
msg = 'Non "send" type found in coinbase deposit/withdrawal processing'
assert raw_data['type'] == 'send', msg
movement_category = AssetMovementCategory.WITHDRAWAL
# Can't see the fee being charged from the "send" resource
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
# Fees dont appear in the docs but from an experiment of sending ETH
# to an address from coinbase there is the network fee in the response
fee = Fee(ZERO)
raw_network = raw_data.get('network', None)
if raw_network:
raw_fee = raw_network.get('transaction_fee', None)
if raw_fee:
# Since this is a withdrawal the fee should be the same as the moved asset
if asset != asset_from_coinbase(raw_fee['currency'],
time=timestamp):
# If not we set ZERO fee and ignore
log.error(
f'In a coinbase withdrawal of {asset.identifier} the fee'
f'is denoted in {raw_fee["currency"]}', )
else:
fee = deserialize_fee(raw_fee['amount'])
if 'network' in raw_data:
transaction_id = get_key_if_has_val(
raw_data['network'], 'hash')
if 'to' in raw_data:
address = deserialize_asset_movement_address(
raw_data['to'], 'address', asset)
else:
movement_category = deserialize_asset_movement_category(
raw_data['resource'])
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
fee = deserialize_fee(raw_data['fee']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
return AssetMovement(
location=Location.COINBASE,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def _import_cryptocom_swap(self, data: Any) -> None:
"""Look for swapping events and handle them as trades.
Notice: Crypto.com csv export gathers all swapping entries (`lockup_swap_*`,
`crypto_wallet_swap_*`, ...) into one entry named `dynamic_coin_swap_*`.
This method looks for `dynamic_coin_swap_debited` and `dynamic_coin_swap_credited`
entries using the same timestamp to handle them as one trade.
"""
swapping_rows: Dict[Any, Dict[str, Any]] = {}
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == 'dynamic_coin_swap_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in swapping_rows:
swapping_rows[timestamp] = {}
swapping_rows[timestamp]['debited'] = row
elif row['Transaction Kind'] == 'dynamic_coin_swap_credited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in swapping_rows:
swapping_rows[timestamp] = {}
swapping_rows[timestamp]['credited'] = row
for timestamp in swapping_rows:
credited_row = swapping_rows[timestamp]['credited']
debited_row = swapping_rows[timestamp]['debited']
if credited_row is not None and debited_row is not None:
notes = 'Coin Swap\nSource: crypto.com (CSV import)'
# No fees here since it's coin swapping
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = Asset(credited_row['Currency'])
quote_asset = Asset(debited_row['Currency'])
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(
credited_row['Amount'])
quote_amount_sold = deserialize_asset_amount(
debited_row['Amount'])
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=TradeType.BUY,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info until (Nov 2020) on crypto.com
# fees are not displayed in the export data
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'referral_bonus',
'crypto_earn_interest_paid',
'referral_card_cashback',
'card_cashback_reverted',
'reimbursement',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = Asset(to_currency)
quote_asset = Asset(currency)
if quote_asset is None:
raise DeserializationError(
'Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = Asset(currency)
quote_asset = Asset(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'crypto_withdrawal' or row_type == 'crypto_deposit':
if row_type == 'crypto_withdrawal':
category = AssetMovementCategory.WITHDRAWAL
amount = deserialize_asset_amount_force_positive(
csv_row['Amount'])
else:
category = AssetMovementCategory.DEPOSIT
amount = deserialize_asset_amount(csv_row['Amount'])
asset = Asset(csv_row['Currency'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in (
'crypto_earn_program_created',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'lockup_swap_rebate',
'dynamic_coin_swap_bonus_exchange_deposit',
# we don't handle cryto.com exchange yet
'crypto_to_exchange_transfer',
'exchange_to_crypto_transfer',
# supercharger actions
'supercharger_deposit',
'supercharger_withdrawal',
# already handled using _import_cryptocom_double_entries
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dust_conversion_debited',
'dust_conversion_credited',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCryptocomEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry', )
def _consume_cointracking_entry(self, csv_row: List[str]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row[1] # Type
timestamp = deserialize_timestamp_from_date(
date=csv_row[9],
formatstr='%d.%m.%Y %H:%M',
location='cointracking.info',
)
notes = csv_row[8]
location = exchange_row_to_location(csv_row[6])
if row_type == 'Gift/Tip' or row_type == 'Trade':
base_asset = Asset(csv_row[3])
quote_asset = None if csv_row[5] == '' else Asset(csv_row[5])
if not quote_asset and row_type != 'Gift/Tip':
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row[2])
if csv_row[4] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row[4])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=Fee(
ZERO), # There are no fees when import from cointracking
fee_currency=base_asset,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row[2])
asset = Asset(csv_row[3])
else:
amount = deserialize_asset_amount(csv_row[4])
asset = Asset(csv_row[5])
asset_movement = AssetMovement(
location=location,
category=category,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=Fee(ZERO),
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = Asset(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = Asset(csv_row['Cur.Buy'])
quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(
csv_row['Cur.Sell'])
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = Asset(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount_force_positive(
csv_row['Sell'])
asset = Asset(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def query_grant_history_period(
self,
grant_id: int,
grant_created_on: Optional[Timestamp],
from_timestamp: Timestamp,
to_timestamp: Timestamp,
) -> Tuple[List[LedgerAction], Optional[Timestamp]]:
transactions: List[Dict[str, Any]] = []
if grant_created_on is None:
query_str = (
f'https://gitcoin.co/api/v0.1/grants/contributions_rec_report/'
f'?id={grant_id}&from_timestamp=2017-09-25&to_timestamp=2017-09-25'
)
result = self._single_grant_api_query(query_str)
try:
grant_created_on = deserialize_timestamp_from_date(
date=result['metadata']['created_on'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin API',
skip_milliseconds=True,
)
from_timestamp = max(grant_created_on, from_timestamp)
grant_name = result['metadata']['grant_name']
self.db_ledger.set_gitcoin_grant_metadata(
grant_id=grant_id,
name=grant_name,
created_on=grant_created_on,
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
log.error(
f'Unexpected data encountered during deserialization of gitcoin api '
f'query: {result}. Error was: {msg}', )
# continue with the given from_timestamp
step_to_timestamp = min(from_timestamp + MONTH_IN_SECONDS,
to_timestamp)
while from_timestamp != step_to_timestamp:
transactions.extend(
self.query_grant_history_period30d(
grant_id=grant_id,
from_ts=from_timestamp,
to_ts=Timestamp(step_to_timestamp),
), )
from_timestamp = Timestamp(step_to_timestamp)
step_to_timestamp = min(step_to_timestamp + MONTH_IN_SECONDS,
to_timestamp)
# Check if any of the clr_payouts are in the range
if self.clr_payouts:
for payout in self.clr_payouts:
timestamp = deserialize_timestamp_from_date(
date=payout['timestamp'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin API',
skip_milliseconds=True,
)
if from_timestamp <= timestamp <= to_timestamp:
round_num = payout.pop('round')
payout['clr_round'] = round_num
transactions.append(payout)
actions = []
for transaction in transactions:
try:
action = _deserialize_transaction(grant_id=grant_id,
rawtx=transaction)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.db.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a gitcoin '
'api query. Check logs for details.', )
log.error(
f'Unexpected data encountered during deserialization of gitcoin api '
f'query: {transaction}. Error was: {msg}', )
continue
except UnknownAsset as e:
self.db.msg_aggregator.add_warning(
f'Found unknown asset {str(e)} in a gitcoin api event transaction. '
'Ignoring it.', )
continue
except ZeroGitcoinAmount:
log.warning(
f'Found gitcoin event with 0 amount for grant {grant_id}. Ignoring'
)
continue
actions.append(action)
return actions, grant_created_on
