def _deserialize_transaction(grant_id: int, rawtx: Dict[str, Any]) -> LedgerAction: """May raise: - DeserializationError - KeyError - UnknownAsset """ timestamp = deserialize_timestamp_from_date( date=rawtx['timestamp'], formatstr='%Y-%m-%dT%H:%M:%S', location='Gitcoin API', skip_milliseconds=True, ) asset = get_gitcoin_asset(symbol=rawtx['asset'], token_address=rawtx['token_address']) raw_amount = deserialize_int_from_str(symbol=rawtx['amount'], location='gitcoin api') amount = asset_normalized_value(raw_amount, asset) if amount == ZERO: raise ZeroGitcoinAmount() # let's use gitcoin's calculated rate for now since they include it in the response usd_value = Price( ZERO) if rawtx['usd_value'] is None else deserialize_price( rawtx['usd_value']) # noqa: E501 rate = Price(ZERO) if usd_value == ZERO else Price(usd_value / amount) raw_txid = rawtx['tx_hash'] tx_type, tx_id = process_gitcoin_txid(key='tx_hash', entry=rawtx) # until we figure out if we can use it https://github.com/gitcoinco/web/issues/9255#issuecomment-874537144 # noqa: E501 clr_round = _calculate_clr_round(timestamp, rawtx) notes = f'Gitcoin grant {grant_id} event' if not clr_round else f'Gitcoin grant {grant_id} event in clr_round {clr_round}' # noqa: E501 return LedgerAction( identifier=1, # whatever -- does not end up in the DB timestamp=timestamp, action_type=LedgerActionType.DONATION_RECEIVED, location=Location.GITCOIN, amount=AssetAmount(amount), asset=asset, rate=rate, rate_asset=A_USD, link=raw_txid, notes=notes, extra_data=GitcoinEventData( tx_id=tx_id, grant_id=grant_id, clr_round=clr_round, tx_type=tx_type, ), )
def trade_from_coinbase(raw_trade: Dict[str, Any]) -> Optional[Trade]: """Turns a coinbase transaction into a rotkehlchen Trade. https://developers.coinbase.com/api/v2?python#buys If the coinbase transaction is not a trade related transaction returns None Throws: - UnknownAsset due to Asset instantiation - DeserializationError due to unexpected format of dict entries - KeyError due to dict entires missing an expected entry """ if raw_trade['status'] != 'completed': # We only want to deal with completed trades return None if raw_trade['instant']: raw_time = raw_trade['created_at'] else: raw_time = raw_trade['payout_at'] timestamp = deserialize_timestamp_from_date(raw_time, 'iso8601', 'coinbase') trade_type = deserialize_trade_type(raw_trade['resource']) tx_amount = deserialize_asset_amount(raw_trade['amount']['amount']) tx_asset = asset_from_coinbase(raw_trade['amount']['currency'], time=timestamp) native_amount = deserialize_asset_amount(raw_trade['subtotal']['amount']) native_asset = asset_from_coinbase(raw_trade['subtotal']['currency'], time=timestamp) # in coinbase you are buying/selling tx_asset for native_asset pair = TradePair(f'{tx_asset.identifier}_{native_asset.identifier}') amount = tx_amount # The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency rate = Price(native_amount / tx_amount) fee_amount = deserialize_fee(raw_trade['fee']['amount']) fee_asset = asset_from_coinbase(raw_trade['fee']['currency'], time=timestamp) return Trade( timestamp=timestamp, location=Location.COINBASE, pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=fee_amount, fee_currency=fee_asset, link=str(raw_trade['id']), )
def _read_asset_movements(self, filepath: str) -> List[AssetMovement]: """Reads a csv account type report and extracts the AssetMovements""" with open(filepath, newline='') as csvfile: reader = csv.DictReader(csvfile) movements = [] for row in reader: try: if row['type'] in ('withdrawal', 'deposit'): timestamp = deserialize_timestamp_from_date( row['time'], 'iso8601', 'coinbasepro', ) asset = Asset(row['amount/balance unit']) movements.append(AssetMovement( location=Location.COINBASEPRO, category=deserialize_asset_movement_category(row['type']), address=None, # can't get it from csv data transaction_id=None, # can't get it from csv data timestamp=timestamp, asset=asset, amount=deserialize_asset_amount_force_positive(row['amount']), fee_asset=asset, # I don't see any fee in deposit withdrawals in coinbasepro fee=Fee(ZERO), link=str(row['transfer id']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring its deposit/withdrawal.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a Coinbasepro deposit/withdrawal. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a coinbasepro deposit/withdrawal.', raw_asset_movement=row, error=msg, ) continue return movements
def _consume_blockfi_trade(self, csv_row: Dict[str, Any]) -> None: """ Consume the file containing only trades from BlockFi. As per my investigations (@yabirgb) this file can only contain confirmed trades. - UnknownAsset - DeserializationError """ timestamp = deserialize_timestamp_from_date( date=csv_row['Date'], formatstr='%Y-%m-%d %H:%M:%S', location='BlockFi', ) buy_asset = symbol_to_asset_or_token(csv_row['Buy Currency']) buy_amount = deserialize_asset_amount(csv_row['Buy Quantity']) sold_asset = symbol_to_asset_or_token(csv_row['Sold Currency']) sold_amount = deserialize_asset_amount(csv_row['Sold Quantity']) if sold_amount == ZERO: log.debug( f'Ignoring BlockFi trade with sold_amount equal to zero. {csv_row}' ) return rate = Price(buy_amount / sold_amount) trade = Trade( timestamp=timestamp, location=Location.BLOCKFI, base_asset=buy_asset, quote_asset=sold_asset, trade_type=TradeType.BUY, amount=buy_amount, rate=rate, fee=None, # BlockFI doesn't provide this information fee_currency=None, link='', notes=csv_row['Type'], ) self.db.add_trades([trade])
def trade_from_ftx(raw_trade: Dict[str, Any]) -> Optional[Trade]: """Turns an FTX transaction into a rotki Trade. May raise: - UnknownAsset due to Asset instantiation - DeserializationError due to unexpected format of dict entries - KeyError due to dict entries missing an expected key """ # In the case of perpetuals and futures this fields can be None if raw_trade.get('baseCurrency', None) is None: return None if raw_trade.get('quoteCurrency', None) is None: return None timestamp = deserialize_timestamp_from_date(raw_trade['time'], 'iso8601', 'FTX') trade_type = TradeType.deserialize(raw_trade['side']) base_asset = asset_from_ftx(raw_trade['baseCurrency']) quote_asset = asset_from_ftx(raw_trade['quoteCurrency']) amount = deserialize_asset_amount(raw_trade['size']) rate = deserialize_price(raw_trade['price']) fee_currency = asset_from_ftx(raw_trade['feeCurrency']) fee = deserialize_fee(raw_trade['fee']) return Trade( timestamp=timestamp, location=Location.FTX, base_asset=base_asset, quote_asset=quote_asset, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, link=str(raw_trade['id']), )
def _consume_blockfi_entry(self, csv_row: Dict[str, Any]) -> None: """ Process entry for BlockFi transaction history. Trades for this file are ignored and istead should be extracted from the file containing only trades. This method can raise: - UnsupportedBlockFiEntry - UnknownAsset - DeserializationError """ if len(csv_row['Confirmed At']) != 0: timestamp = deserialize_timestamp_from_date( date=csv_row['Confirmed At'], formatstr='%Y-%m-%d %H:%M:%S', location='BlockFi', ) else: log.debug(f'Ignoring unconfirmed BlockFi entry {csv_row}') return asset = symbol_to_asset_or_token(csv_row['Cryptocurrency']) amount = deserialize_asset_amount_force_positive(csv_row['Amount']) entry_type = csv_row['Transaction Type'] # BlockFI doesn't provide information about fees fee = Fee(ZERO) fee_asset = A_USD # Can be whatever if entry_type in ('Deposit', 'Wire Deposit', 'ACH Deposit'): asset_movement = AssetMovement( location=Location.BLOCKFI, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_asset, link='', ) self.db.add_asset_movements([asset_movement]) elif entry_type in ('Withdrawal', 'Wire Withdrawal', 'ACH Withdrawal'): asset_movement = AssetMovement( location=Location.BLOCKFI, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_asset, link='', ) self.db.add_asset_movements([asset_movement]) elif entry_type == 'Withdrawal Fee': action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.EXPENSE, location=Location.BLOCKFI, amount=amount, asset=asset, rate=None, rate_asset=None, link=None, notes=f'{entry_type} from BlockFi', ) self.db_ledger.add_ledger_action(action) elif entry_type in ('Interest Payment', 'Bonus Payment', 'Referral Bonus'): action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.INCOME, location=Location.BLOCKFI, amount=amount, asset=asset, rate=None, rate_asset=None, link=None, notes=f'{entry_type} from BlockFi', ) self.db_ledger.add_ledger_action(action) elif entry_type == 'Trade': pass else: raise UnsupportedCSVEntry( f'Unsuported entry {entry_type}. Data: {csv_row}')
def _import_cryptocom_associated_entries(self, data: Any, tx_kind: str) -> None: """Look for events that have associated entries and handle them as trades. This method looks for `*_debited` and `*_credited` entries using the same timestamp to handle them as one trade. Known kind: 'dynamic_coin_swap' or 'dust_conversion' May raise: - UnknownAsset if an unknown asset is encountered in the imported files - KeyError if a row contains unexpected data entries """ multiple_rows: Dict[Any, Dict[str, Any]] = {} investments_deposits: Dict[str, List[Any]] = defaultdict(list) investments_withdrawals: Dict[str, List[Any]] = defaultdict(list) debited_row = None credited_row = None for row in data: if row['Transaction Kind'] == f'{tx_kind}_debited': timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ) if timestamp not in multiple_rows: multiple_rows[timestamp] = {} if 'debited' not in multiple_rows[timestamp]: multiple_rows[timestamp]['debited'] = [] multiple_rows[timestamp]['debited'].append(row) elif row['Transaction Kind'] == f'{tx_kind}_credited': # They only is one credited row timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ) if timestamp not in multiple_rows: multiple_rows[timestamp] = {} multiple_rows[timestamp]['credited'] = row elif row['Transaction Kind'] == f'{tx_kind}_deposit': asset = row['Currency'] investments_deposits[asset].append(row) elif row['Transaction Kind'] == f'{tx_kind}_withdrawal': asset = row['Currency'] investments_withdrawals[asset].append(row) for timestamp in multiple_rows: # When we convert multiple assets dust to CRO # in one time, it will create multiple debited rows with # the same timestamp debited_rows = multiple_rows[timestamp]['debited'] credited_row = multiple_rows[timestamp]['credited'] total_debited_usd = functools.reduce( lambda acc, row: acc + deserialize_asset_amount(row[ 'Native Amount (in USD)']), debited_rows, ZERO, ) # If the value of the transaction is too small (< 0,01$), # crypto.com will display 0 as native amount # if we have multiple debited rows, we can't import them # since we can't compute their dedicated rates, so we skip them if len(debited_rows) > 1 and total_debited_usd == 0: return if credited_row is not None and len(debited_rows) != 0: for debited_row in debited_rows: description = credited_row['Transaction Description'] notes = f'{description}\nSource: crypto.com (CSV import)' # No fees here fee = Fee(ZERO) fee_currency = A_USD base_asset = symbol_to_asset_or_token( credited_row['Currency']) quote_asset = symbol_to_asset_or_token( debited_row['Currency']) part_of_total = ( FVal(1) if len(debited_rows) == 1 else deserialize_asset_amount( debited_row["Native Amount (in USD)"], ) / total_debited_usd) quote_amount_sold = deserialize_asset_amount( debited_row['Amount'], ) * part_of_total base_amount_bought = deserialize_asset_amount( credited_row['Amount'], ) * part_of_total rate = Price(abs(base_amount_bought / quote_amount_sold)) trade = Trade( timestamp=timestamp, location=Location.CRYPTOCOM, base_asset=base_asset, quote_asset=quote_asset, trade_type=TradeType.BUY, amount=AssetAmount(base_amount_bought), rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) # Compute investments profit if len(investments_withdrawals) != 0: for asset in investments_withdrawals: asset_object = symbol_to_asset_or_token(asset) if asset not in investments_deposits: log.error( f'Investment withdrawal without deposit at crypto.com. Ignoring ' f'staking info for asset {asset_object}', ) continue # Sort by date in ascending order withdrawals_rows = sorted( investments_withdrawals[asset], key=lambda x: deserialize_timestamp_from_date( date=x['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ), ) investments_rows = sorted( investments_deposits[asset], key=lambda x: deserialize_timestamp_from_date( date=x['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ), ) last_date = Timestamp(0) for withdrawal in withdrawals_rows: withdrawal_date = deserialize_timestamp_from_date( date=withdrawal['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ) amount_deposited = ZERO for deposit in investments_rows: deposit_date = deserialize_timestamp_from_date( date=deposit['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ) if last_date < deposit_date <= withdrawal_date: # Amount is negative amount_deposited += deserialize_asset_amount( deposit['Amount']) amount_withdrawal = deserialize_asset_amount( withdrawal['Amount']) # Compute profit profit = amount_withdrawal + amount_deposited if profit >= ZERO: last_date = withdrawal_date action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=withdrawal_date, action_type=LedgerActionType.INCOME, location=Location.CRYPTOCOM, amount=AssetAmount(profit), asset=asset_object, rate=None, rate_asset=None, link=None, notes=f'Stake profit for asset {asset}', ) self.db_ledger.add_ledger_action(action)
def _import_cryptocom_double_entries(self, data: Any, double_type: str) -> None: """Look for events that have double entries and handle them as trades. This method looks for `*_debited` and `*_credited` entries using the same timestamp to handle them as one trade. Known double_type: 'dynamic_coin_swap' or 'dust_conversion' """ double_rows: Dict[Any, Dict[str, Any]] = {} debited_row = None credited_row = None for row in data: if row['Transaction Kind'] == f'{double_type}_debited': timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) if timestamp not in double_rows: double_rows[timestamp] = {} double_rows[timestamp]['debited'] = row elif row['Transaction Kind'] == f'{double_type}_credited': timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) if timestamp not in double_rows: double_rows[timestamp] = {} double_rows[timestamp]['credited'] = row for timestamp in double_rows: credited_row = double_rows[timestamp]['credited'] debited_row = double_rows[timestamp]['debited'] if credited_row is not None and debited_row is not None: description = credited_row['Transaction Description'] notes = f'{description}\nSource: crypto.com (CSV import)' # No fees here fee = Fee(ZERO) fee_currency = A_USD base_asset = Asset(credited_row['Currency']) quote_asset = Asset(debited_row['Currency']) pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount( credited_row['Amount']) quote_amount_sold = deserialize_asset_amount( debited_row['Amount']) rate = Price(abs(base_amount_bought / quote_amount_sold)) trade = Trade( timestamp=timestamp, location=Location.CRYPTOCOM, pair=pair, trade_type=TradeType.BUY, amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade])
def _deserialize_ledger_action( self, raw_data: Dict[str, Any]) -> Optional[LedgerAction]: """Processes a single transaction from coinbase and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if raw_data.get('status', '') != 'completed': return None payout_date = raw_data.get('payout_at', None) if payout_date: timestamp = deserialize_timestamp_from_date( payout_date, 'iso8601', 'coinbase') else: timestamp = deserialize_timestamp_from_date( get_key_if_has_val(raw_data, 'created_at'), 'iso8601', 'coinbase', ) if 'type' in raw_data: # The parent method filtered with 'from' attribute, so it is from another user. # https://developers.coinbase.com/api/v2?python#transaction-resource action_type = LedgerActionType.INCOME if raw_data.get('type', '') not in ('send', 'inflation_reward'): msg = ('Non "send" or "inflation_reward" type ' 'found in coinbase transactions processing') raise DeserializationError(msg) amount_data = raw_data.get('amount', {}) amount = deserialize_asset_amount(amount_data['amount']) asset = asset_from_coinbase(amount_data['currency'], time=timestamp) native_amount_data = raw_data.get('native_amount', {}) native_amount = deserialize_asset_amount( native_amount_data['amount']) native_asset = asset_from_coinbase( native_amount_data['currency']) rate = ZERO if amount_data and native_amount_data and native_amount and amount != ZERO: rate = native_amount / amount if 'details' in raw_data and 'title' in raw_data['details'] \ and 'subtitle' in raw_data['details'] and 'header' in raw_data['details']: details = raw_data.get('details', {}) notes = (f"{details.get('title', '')} " f"{details.get('subtitle', '')} " f"{details.get('header', '')}") else: notes = '' return LedgerAction(identifier=0, location=Location.COINBASE, action_type=action_type, timestamp=timestamp, asset=asset, amount=amount, rate=Price(rate), rate_asset=native_asset, link=str(raw_data['id']), notes=notes) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found coinbase transaction with unknown asset ' f'{e.asset_name}. Ignoring it.', ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found coinbase transaction with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a coinbase ' 'ledger action. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of coinbase ' f'ledger action {raw_data}. Error was: {msg}', ) return None
def _deserialize_asset_movement( self, raw_data: Dict[str, Any], movement_type: AssetMovementCategory, ) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from FTX and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if raw_data['status'] not in ('complete', 'confirmed'): return None timestamp = deserialize_timestamp_from_date( raw_data['time'], 'iso8601', 'FTX') amount = deserialize_asset_amount_force_positive(raw_data['size']) asset = asset_from_ftx(raw_data['coin']) fee = Fee(ZERO) movement_category = movement_type if raw_data.get('fee', None) is not None: fee = deserialize_fee(raw_data['fee']) address = raw_data.get('address', None) if isinstance(address, dict): address = raw_data['address'].get('address', None) transaction_id = raw_data.get('txid', None) return AssetMovement( location=Location.FTX, category=movement_category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(raw_data['id']), ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found FTX deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found FTX deposit/withdrawal with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of an FTX ' 'asset movement. Check logs for details and open a bug report.', ) log.error( 'Error processing FTX trade', trade=raw_data, error=msg, ) return None
def _api_query( self, endpoint: str, start_time: Optional[Timestamp] = None, end_time: Optional[Timestamp] = None, limit: int = PAGINATION_LIMIT, paginate: bool = True, ) -> Union[List[Dict[str, Any]], Dict[str, List[Any]], Dict[str, Any]]: """Query FTX endpoint and retrieve all available information if pagination is requested. In case of paginate being set to False only one request is made. Can raise: - RemoteError """ if not paginate: final_data_no_pag = self._make_request( endpoint=endpoint, limit=limit, start_time=start_time, end_time=end_time, ) return final_data_no_pag # If there is pagination we follow the example from the official ftx python example # https://github.com/ftexchange/ftx/blob/master/rest/client.py#L163 # In this case the strategy is a while loop leaving fixed the start_time (lower bound) # and decreasing end time (the upper bound) until we fetch all the available information new_end_time = end_time ids = set() final_data: List[Dict[str, Any]] = [] while True: step = self._make_request( endpoint=endpoint, limit=limit, start_time=start_time, end_time=new_end_time, ) if not isinstance(step, list): raise RemoteError( f'FTX pagination returned something different to a list for route {endpoint} ' f'with start_time {start_time} and end_time {end_time}. Result was {step}.', ) # remove possible duplicates deduped = [ r for r in step if 'id' in r.keys() and r['id'] not in ids ] ids |= {r['id'] for r in deduped} final_data.extend(deduped) if len(step) == 0: break # Avoid deserialization error if there is a bad date times = [] for t in step: try: times.append( deserialize_timestamp_from_date( t['time'], 'iso8601', 'ftx')) except (DeserializationError, KeyError): continue if len(times) != 0: new_end_time = min(times) else: self.msg_aggregator.add_error( f'Error processing FTX trade history. Query step ' f'returned invalid entries when trying pagination for endpoint ' f'{endpoint} with start_time {start_time}, end_time {end_time} ' f'and page limit of {limit}.', ) break if len(step) < limit: break return final_data
def retrieve_location_data(data_dir: Path) -> Optional[GeolocationData]: """This functions tries to get the country of the user based on the ip. To do that it makes use of an open ip to country database and tries to obtain the ip ussing UPnP protocol. If UpNP fails we just return None **IMPORTANT:** The ip never leaves the user's machine. It's all calculated locally. """ geoip_dir = data_dir / 'misc' geoip_dir.mkdir(parents=True, exist_ok=True) # get latest database version metadata_query_failed = False try: response = requests.get( url='https://api.github.com/repos/geoacumen/geoacumen-country/branches/master', timeout=DEFAULT_TIMEOUT_TUPLE, ) data = response.json() date = deserialize_timestamp_from_date( date=data['commit']['commit']['author']['date'], formatstr='%Y-%m-%dT%H:%M:%S', location='Analytics', ) except requests.exceptions.RequestException as e: log.debug(f'Failed to get metadata information for geoip file. {str(e)}') metadata_query_failed = True except (DeserializationError, JSONDecodeError) as e: log.debug(f'Failed to deserialize date in metadata information for geoip file. {str(e)}') metadata_query_failed = True except KeyError as e: log.debug(f'Github response for geoip file had missing key {str(e)}') metadata_query_failed = True if metadata_query_failed: old_files = list(geoip_dir.glob('*.mmdb')) if len(old_files) == 0: return None filename = old_files[0] else: filename = geoip_dir / f'geoip-{date}.mmdb' if not filename.is_file(): # Remove old files files = geoip_dir.glob('*.*') for f in files: f.unlink() # Download latest version try: response = requests.get( url='https://github.com/geoacumen/geoacumen-country/raw/master/Geoacumen-Country.mmdb', # noqa: E501 timeout=DEFAULT_TIMEOUT_TUPLE, stream=True, ) except requests.exceptions.RequestException as e: log.debug(f'Failed to download geoip database file. {str(e)}') return None with open(filename, 'wb') as outfile: response.raw.decode_content = True shutil.copyfileobj(response.raw, outfile) # get user ip try: u = miniupnpc.UPnP() u.discoverdelay = 200 u.discover() u.selectigd() user_ip = u.externalipaddress() except Exception as e: # pylint: disable=broad-except # can raise base exception so we catch it log.debug(f'Failed to get ip via UPnP for analytics. {str(e)}') return None try: with maxminddb.open_database(filename) as reader: data = reader.get(user_ip) location = data['country']['iso_code'] if location == 'None': return None return GeolocationData(country_code=location) except maxminddb.errors.InvalidDatabaseError as e: filename.unlink() log.debug(f'Failed to read database {str(e)}') except ValueError as e: log.debug(f'Wrong ip search {str(e)}') return None
def _consume_grant_entry(self, entry: Dict[str, Any]) -> Optional[LedgerAction]: """ Consumes a grant entry from the CSV and turns it into a LedgerAction May raise: - DeserializationError - KeyError - UnknownAsset """ if entry['Type'] != 'grant': return None timestamp = deserialize_timestamp_from_date( date=entry['date'], formatstr='%Y-%m-%dT%H:%M:%S', location='Gitcoin CSV', skip_milliseconds=True, ) usd_value = deserialize_asset_amount(entry['Value In USD']) asset = get_asset_by_symbol(entry['token_name']) if asset is None: raise UnknownAsset(entry['token_name']) token_amount = deserialize_asset_amount(entry['token_value']) if token_amount == ZERO: # try to make up for https://github.com/gitcoinco/web/issues/9213 price = query_usd_price_zero_if_error( asset=asset, time=timestamp, location=f'Gitcoin CSV entry {entry["txid"]}', msg_aggregator=self.db.msg_aggregator, ) if price == ZERO: self.db.msg_aggregator.add_warning( f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} ' f'due to amount being zero and inability to find price. Skipping.', ) return None # calculate the amount from price and value token_amount = usd_value / price # type: ignore match = self.grantid_re.search(entry['url']) if match is None: self.db.msg_aggregator.add_warning( f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} ' f'due to inability to read grant id. Skipping.', ) return None grant_id = int(match.group(1)) rate = Price(usd_value / token_amount) raw_txid = entry['txid'] tx_type, tx_id = process_gitcoin_txid(key='txid', entry=entry) return LedgerAction( identifier=1, # whatever does not go in the DB timestamp=timestamp, action_type=LedgerActionType.DONATION_RECEIVED, location=Location.GITCOIN, amount=token_amount, asset=asset, rate=rate, rate_asset=A_USD, # let's use the rate gitcoin calculated link=raw_txid, notes=f'Gitcoin grant {grant_id} event', extra_data=GitcoinEventData( tx_id=tx_id, grant_id=grant_id, clr_round=None, # can't get round from CSV tx_type=tx_type, ), )
def _import_cryptocom_associated_entries(self, data: Any, tx_kind: str) -> None: """Look for events that have associated entries and handle them as trades. This method looks for `*_debited` and `*_credited` entries using the same timestamp to handle them as one trade. Known kind: 'dynamic_coin_swap' or 'dust_conversion' """ multiple_rows: Dict[Any, Dict[str, Any]] = {} debited_row = None credited_row = None for row in data: if row['Transaction Kind'] == f'{tx_kind}_debited': timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) if timestamp not in multiple_rows: multiple_rows[timestamp] = {} if 'debited' not in multiple_rows[timestamp]: multiple_rows[timestamp]['debited'] = [] multiple_rows[timestamp]['debited'].append(row) elif row['Transaction Kind'] == f'{tx_kind}_credited': # They only is one credited row timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) if timestamp not in multiple_rows: multiple_rows[timestamp] = {} multiple_rows[timestamp]['credited'] = row for timestamp in multiple_rows: # When we convert multiple assets dust to CRO # in one time, it will create multiple debited rows with # the same timestamp debited_rows = multiple_rows[timestamp]['debited'] credited_row = multiple_rows[timestamp]['credited'] total_debited_usd = functools.reduce( lambda acc, row: acc + deserialize_asset_amount(row[ 'Native Amount (in USD)']), debited_rows, FVal('0'), ) # If the value of the transaction is too small (< 0,01$), # crypto.com will display 0 as native amount # if we have multiple debited rows, we can't import them # since we can't compute their dedicated rates, so we skip them if len(debited_rows) > 1 and total_debited_usd == 0: return if credited_row is not None and len(debited_rows) != 0: for debited_row in debited_rows: description = credited_row['Transaction Description'] notes = f'{description}\nSource: crypto.com (CSV import)' # No fees here fee = Fee(ZERO) fee_currency = A_USD base_asset = Asset(credited_row['Currency']) quote_asset = Asset(debited_row['Currency']) pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') part_of_total = ( FVal(1) if len(debited_rows) == 1 else deserialize_asset_amount( debited_row["Native Amount (in USD)"], ) / total_debited_usd) quote_amount_sold = deserialize_asset_amount( debited_row['Amount'], ) * part_of_total base_amount_bought = deserialize_asset_amount( credited_row['Amount'], ) * part_of_total rate = Price(abs(base_amount_bought / quote_amount_sold)) trade = Trade( timestamp=timestamp, location=Location.CRYPTOCOM, pair=pair, trade_type=TradeType.BUY, amount=AssetAmount(base_amount_bought), rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade])
def _consume_nexo(self, csv_row: Dict[str, Any]) -> None: """ Consume CSV file from NEXO. This method can raise: - UnsupportedNexoEntry - UnknownAsset - DeserializationError """ ignored_entries = ('ExchangeToWithdraw', 'DepositToExchange') if 'rejected' not in csv_row['Details']: timestamp = deserialize_timestamp_from_date( date=csv_row['Date / Time'], formatstr='%Y-%m-%d %H:%M', location='NEXO', ) else: log.debug(f'Ignoring rejected nexo entry {csv_row}') return asset = symbol_to_asset_or_token(csv_row['Currency']) amount = deserialize_asset_amount_force_positive(csv_row['Amount']) entry_type = csv_row['Type'] transaction = csv_row['Transaction'] if entry_type in ('Deposit', 'ExchangeDepositedOn', 'LockingTermDeposit'): asset_movement = AssetMovement( location=Location.NEXO, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=Fee(ZERO), fee_asset=A_USD, link=transaction, ) self.db.add_asset_movements([asset_movement]) elif entry_type in ('Withdrawal', 'WithdrawExchanged'): asset_movement = AssetMovement( location=Location.NEXO, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=Fee(ZERO), fee_asset=A_USD, link=transaction, ) self.db.add_asset_movements([asset_movement]) elif entry_type == 'Withdrawal Fee': action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.EXPENSE, location=Location.NEXO, amount=amount, asset=asset, rate=None, rate_asset=None, link=None, notes=f'{entry_type} from Nexo', ) self.db_ledger.add_ledger_action(action) elif entry_type in ('Interest', 'Bonus', 'Dividend'): action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.INCOME, location=Location.NEXO, amount=amount, asset=asset, rate=None, rate_asset=None, link=transaction, notes=f'{entry_type} from Nexo', ) self.db_ledger.add_ledger_action(action) elif entry_type in ignored_entries: pass else: raise UnsupportedCSVEntry( f'Unsuported entry {entry_type}. Data: {csv_row}')
def trade_from_conversion(trade_a: Dict[str, Any], trade_b: Dict[str, Any]) -> Optional[Trade]: """Turn information from a conversion into a trade Mary raise: - UnknownAsset due to Asset instantiation - DeserializationError due to unexpected format of dict entries - KeyError due to dict entires missing an expected entry """ # Check that the status is complete if trade_a['status'] != 'completed': return None # Trade b will represent the asset we are converting to if trade_b['amount']['amount'].startswith('-'): trade_a, trade_b = trade_b, trade_a timestamp = deserialize_timestamp_from_date(trade_a['updated_at'], 'iso8601', 'coinbase') tx_amount = AssetAmount( abs(deserialize_asset_amount(trade_a['amount']['amount']))) tx_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp) native_amount = deserialize_asset_amount(trade_b['amount']['amount']) native_asset = asset_from_coinbase(trade_b['amount']['currency'], time=timestamp) amount = tx_amount # The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency rate = Price(native_amount / tx_amount) # Obtain fee amount in the native currency using data from both trades amount_after_fee = deserialize_asset_amount( trade_b['native_amount']['amount']) amount_before_fee = deserialize_asset_amount( trade_a['native_amount']['amount']) # amount_after_fee + amount_before_fee is a negative amount and the fee needs to be positive conversion_native_fee_amount = abs(amount_after_fee + amount_before_fee) if ZERO not in (tx_amount, conversion_native_fee_amount, amount_before_fee, amount_after_fee): # To get the asset in which the fee is nominated we pay attention to the creation # date of each event. As per our hypothesis the fee is nominated in the asset # for which the first transaction part was initialized time_created_a = deserialize_timestamp_from_date( date=trade_a['created_at'], formatstr='iso8601', location='coinbase', ) time_created_b = deserialize_timestamp_from_date( date=trade_b['created_at'], formatstr='iso8601', location='coinbase', ) if time_created_a < time_created_b: # We have the fee amount in the native currency. To get it in the # converted asset we have to get the rate asset_native_rate = tx_amount / abs(amount_before_fee) fee_amount = Fee(conversion_native_fee_amount * asset_native_rate) fee_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp) else: trade_b_amount = abs( deserialize_asset_amount(trade_b['amount']['amount'])) asset_native_rate = trade_b_amount / abs(amount_after_fee) fee_amount = Fee(conversion_native_fee_amount * asset_native_rate) fee_asset = asset_from_coinbase(trade_b['amount']['currency'], time=timestamp) else: fee_amount = Fee(ZERO) fee_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp) return Trade( timestamp=timestamp, location=Location.COINBASE, # in coinbase you are buying/selling tx_asset for native_asset base_asset=tx_asset, quote_asset=native_asset, trade_type=TradeType.SELL, amount=amount, rate=rate, fee=fee_amount, fee_currency=fee_asset, link=str(trade_a['trade']['id']), )
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None: """Consumes a cryptocom entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCryptocomEntry if importing of this entry is not supported. - KeyError if the an expected CSV key is missing - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row['Transaction Kind'] timestamp = deserialize_timestamp_from_date( date=csv_row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) description = csv_row['Transaction Description'] notes = f'{description}\nSource: crypto.com (CSV import)' # No fees info for now (Aug 2020) on crypto.com, so we put 0 fees fee = Fee(ZERO) fee_currency = A_USD # whatever (used only if there is no fee) if row_type in ( 'crypto_purchase', 'crypto_exchange', 'referral_gift', 'crypto_earn_interest_paid', ): # variable mapping to raw data currency = csv_row['Currency'] to_currency = csv_row['To Currency'] native_currency = csv_row['Native Currency'] amount = csv_row['Amount'] to_amount = csv_row['To Amount'] native_amount = csv_row['Native Amount'] trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL if row_type == 'crypto_exchange': # trades crypto to crypto base_asset = Asset(to_currency) quote_asset = Asset(currency) if quote_asset is None: raise DeserializationError( 'Got a trade entry with an empty quote asset') base_amount_bought = deserialize_asset_amount(to_amount) quote_amount_sold = deserialize_asset_amount(amount) else: base_asset = Asset(currency) quote_asset = Asset(native_currency) base_amount_bought = deserialize_asset_amount(amount) quote_amount_sold = deserialize_asset_amount(native_amount) rate = Price(abs(quote_amount_sold / base_amount_bought)) pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') trade = Trade( timestamp=timestamp, location=Location.CRYPTOCOM, pair=pair, trade_type=trade_type, amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type in ( 'crypto_earn_program_created', 'lockup_lock', 'lockup_unlock', 'dynamic_coin_swap_bonus_exchange_deposit', 'crypto_wallet_swap_debited', 'crypto_wallet_swap_credited', 'lockup_swap_debited', 'lockup_swap_credited', 'dynamic_coin_swap_debited', 'dynamic_coin_swap_credited', 'dynamic_coin_swap_bonus_exchange_deposit', ): # those types are ignored because it doesn't affect the wallet balance # or are not handled here return else: raise UnsupportedCryptocomEntry( f'Unknown entrype type "{row_type}" encountered during ' f'cryptocom data import. Ignoring entry', )
def _deserialize_asset_movement( self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from coinbase and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if raw_data['status'] != 'completed': return None payout_date = raw_data.get('payout_at', None) if payout_date: timestamp = deserialize_timestamp_from_date( payout_date, 'iso8601', 'coinbase') else: timestamp = deserialize_timestamp_from_date( raw_data['created_at'], 'iso8601', 'coinbase', ) # Only get address/transaction id for "send" type of transactions address = None transaction_id = None # movement_category: Union[Literal['deposit'], Literal['withdrawal']] if 'type' in raw_data: # Then this should be a "send" which is the way Coinbase uses to send # crypto outside of the exchange # https://developers.coinbase.com/api/v2?python#transaction-resource msg = 'Non "send" type found in coinbase deposit/withdrawal processing' assert raw_data['type'] == 'send', msg movement_category = AssetMovementCategory.WITHDRAWAL # Can't see the fee being charged from the "send" resource amount = deserialize_asset_amount_force_positive( raw_data['amount']['amount']) asset = asset_from_coinbase(raw_data['amount']['currency'], time=timestamp) # Fees dont appear in the docs but from an experiment of sending ETH # to an address from coinbase there is the network fee in the response fee = Fee(ZERO) raw_network = raw_data.get('network', None) if raw_network: raw_fee = raw_network.get('transaction_fee', None) if raw_fee: # Since this is a withdrawal the fee should be the same as the moved asset if asset != asset_from_coinbase(raw_fee['currency'], time=timestamp): # If not we set ZERO fee and ignore log.error( f'In a coinbase withdrawal of {asset.identifier} the fee' f'is denoted in {raw_fee["currency"]}', ) else: fee = deserialize_fee(raw_fee['amount']) if 'network' in raw_data: transaction_id = get_key_if_has_val( raw_data['network'], 'hash') if 'to' in raw_data: address = deserialize_asset_movement_address( raw_data['to'], 'address', asset) else: movement_category = deserialize_asset_movement_category( raw_data['resource']) amount = deserialize_asset_amount_force_positive( raw_data['amount']['amount']) fee = deserialize_fee(raw_data['fee']['amount']) asset = asset_from_coinbase(raw_data['amount']['currency'], time=timestamp) return AssetMovement( location=Location.COINBASE, category=movement_category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(raw_data['id']), ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found coinbase deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found coinbase deposit/withdrawal with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a coinbase ' 'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of coinbase ' f'asset_movement {raw_data}. Error was: {str(e)}', ) return None
def _import_cryptocom_swap(self, data: Any) -> None: """Look for swapping events and handle them as trades. Notice: Crypto.com csv export gathers all swapping entries (`lockup_swap_*`, `crypto_wallet_swap_*`, ...) into one entry named `dynamic_coin_swap_*`. This method looks for `dynamic_coin_swap_debited` and `dynamic_coin_swap_credited` entries using the same timestamp to handle them as one trade. """ swapping_rows: Dict[Any, Dict[str, Any]] = {} debited_row = None credited_row = None for row in data: if row['Transaction Kind'] == 'dynamic_coin_swap_debited': timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) if timestamp not in swapping_rows: swapping_rows[timestamp] = {} swapping_rows[timestamp]['debited'] = row elif row['Transaction Kind'] == 'dynamic_coin_swap_credited': timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) if timestamp not in swapping_rows: swapping_rows[timestamp] = {} swapping_rows[timestamp]['credited'] = row for timestamp in swapping_rows: credited_row = swapping_rows[timestamp]['credited'] debited_row = swapping_rows[timestamp]['debited'] if credited_row is not None and debited_row is not None: notes = 'Coin Swap\nSource: crypto.com (CSV import)' # No fees here since it's coin swapping fee = Fee(ZERO) fee_currency = A_USD base_asset = Asset(credited_row['Currency']) quote_asset = Asset(debited_row['Currency']) pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount( credited_row['Amount']) quote_amount_sold = deserialize_asset_amount( debited_row['Amount']) rate = Price(abs(base_amount_bought / quote_amount_sold)) trade = Trade( timestamp=timestamp, location=Location.CRYPTOCOM, pair=pair, trade_type=TradeType.BUY, amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade])
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None: """Consumes a cryptocom entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCryptocomEntry if importing of this entry is not supported. - KeyError if the an expected CSV key is missing - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row['Transaction Kind'] timestamp = deserialize_timestamp_from_date( date=csv_row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) description = csv_row['Transaction Description'] notes = f'{description}\nSource: crypto.com (CSV import)' # No fees info until (Nov 2020) on crypto.com # fees are not displayed in the export data fee = Fee(ZERO) fee_currency = A_USD # whatever (used only if there is no fee) if row_type in ( 'crypto_purchase', 'crypto_exchange', 'referral_gift', 'referral_bonus', 'crypto_earn_interest_paid', 'referral_card_cashback', 'card_cashback_reverted', 'reimbursement', ): # variable mapping to raw data currency = csv_row['Currency'] to_currency = csv_row['To Currency'] native_currency = csv_row['Native Currency'] amount = csv_row['Amount'] to_amount = csv_row['To Amount'] native_amount = csv_row['Native Amount'] trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL if row_type == 'crypto_exchange': # trades crypto to crypto base_asset = Asset(to_currency) quote_asset = Asset(currency) if quote_asset is None: raise DeserializationError( 'Got a trade entry with an empty quote asset') base_amount_bought = deserialize_asset_amount(to_amount) quote_amount_sold = deserialize_asset_amount(amount) else: base_asset = Asset(currency) quote_asset = Asset(native_currency) base_amount_bought = deserialize_asset_amount(amount) quote_amount_sold = deserialize_asset_amount(native_amount) rate = Price(abs(quote_amount_sold / base_amount_bought)) pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') trade = Trade( timestamp=timestamp, location=Location.CRYPTOCOM, pair=pair, trade_type=trade_type, amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type == 'crypto_withdrawal' or row_type == 'crypto_deposit': if row_type == 'crypto_withdrawal': category = AssetMovementCategory.WITHDRAWAL amount = deserialize_asset_amount_force_positive( csv_row['Amount']) else: category = AssetMovementCategory.DEPOSIT amount = deserialize_asset_amount(csv_row['Amount']) asset = Asset(csv_row['Currency']) asset_movement = AssetMovement( location=Location.CRYPTOCOM, category=category, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=asset, link='', ) self.db.add_asset_movements([asset_movement]) elif row_type in ( 'crypto_earn_program_created', 'lockup_lock', 'lockup_unlock', 'dynamic_coin_swap_bonus_exchange_deposit', 'crypto_wallet_swap_debited', 'crypto_wallet_swap_credited', 'lockup_swap_debited', 'lockup_swap_credited', 'lockup_swap_rebate', 'dynamic_coin_swap_bonus_exchange_deposit', # we don't handle cryto.com exchange yet 'crypto_to_exchange_transfer', 'exchange_to_crypto_transfer', # supercharger actions 'supercharger_deposit', 'supercharger_withdrawal', # already handled using _import_cryptocom_double_entries 'dynamic_coin_swap_debited', 'dynamic_coin_swap_credited', 'dust_conversion_debited', 'dust_conversion_credited', ): # those types are ignored because it doesn't affect the wallet balance # or are not handled here return else: raise UnsupportedCryptocomEntry( f'Unknown entrype type "{row_type}" encountered during ' f'cryptocom data import. Ignoring entry', )
def _consume_cointracking_entry(self, csv_row: List[str]) -> None: """Consumes a cointracking entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCointrackingEntry if importing of this entry is not supported. - IndexError if the CSV file is corrupt - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row[1] # Type timestamp = deserialize_timestamp_from_date( date=csv_row[9], formatstr='%d.%m.%Y %H:%M', location='cointracking.info', ) notes = csv_row[8] location = exchange_row_to_location(csv_row[6]) if row_type == 'Gift/Tip' or row_type == 'Trade': base_asset = Asset(csv_row[3]) quote_asset = None if csv_row[5] == '' else Asset(csv_row[5]) if not quote_asset and row_type != 'Gift/Tip': raise DeserializationError( 'Got a trade entry with an empty quote asset') if quote_asset is None: # Really makes no difference as this is just a gift and the amount is zero quote_asset = A_USD pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount(csv_row[2]) if csv_row[4] != '-': quote_amount_sold = deserialize_asset_amount(csv_row[4]) else: quote_amount_sold = AssetAmount(ZERO) rate = Price(quote_amount_sold / base_amount_bought) trade = Trade( timestamp=timestamp, location=location, pair=pair, trade_type=TradeType. BUY, # It's always a buy during cointracking import amount=base_amount_bought, rate=rate, fee=Fee( ZERO), # There are no fees when import from cointracking fee_currency=base_asset, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type == 'Deposit' or row_type == 'Withdrawal': category = deserialize_asset_movement_category(row_type.lower()) if category == AssetMovementCategory.DEPOSIT: amount = deserialize_asset_amount(csv_row[2]) asset = Asset(csv_row[3]) else: amount = deserialize_asset_amount(csv_row[4]) asset = Asset(csv_row[5]) asset_movement = AssetMovement( location=location, category=category, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=Fee(ZERO), link='', ) self.db.add_asset_movements([asset_movement]) else: raise UnsupportedCointrackingEntry( f'Unknown entrype type "{row_type}" encountered during cointracking ' f'data import. Ignoring entry', )
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None: """Consumes a cointracking entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCointrackingEntry if importing of this entry is not supported. - IndexError if the CSV file is corrupt - KeyError if the an expected CSV key is missing - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row['Type'] timestamp = deserialize_timestamp_from_date( date=csv_row['Date'], formatstr='%d.%m.%Y %H:%M:%S', location='cointracking.info', ) notes = csv_row['Comment'] location = exchange_row_to_location(csv_row['Exchange']) fee = Fee(ZERO) fee_currency = A_USD # whatever (used only if there is no fee) if csv_row['Fee'] != '': fee = deserialize_fee(csv_row['Fee']) fee_currency = Asset(csv_row['Cur.Fee']) if row_type in ('Gift/Tip', 'Trade', 'Income'): base_asset = Asset(csv_row['Cur.Buy']) quote_asset = None if csv_row['Cur.Sell'] == '' else Asset( csv_row['Cur.Sell']) if quote_asset is None and row_type not in ('Gift/Tip', 'Income'): raise DeserializationError( 'Got a trade entry with an empty quote asset') if quote_asset is None: # Really makes no difference as this is just a gift and the amount is zero quote_asset = A_USD pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount(csv_row['Buy']) if csv_row['Sell'] != '-': quote_amount_sold = deserialize_asset_amount(csv_row['Sell']) else: quote_amount_sold = AssetAmount(ZERO) rate = Price(quote_amount_sold / base_amount_bought) trade = Trade( timestamp=timestamp, location=location, pair=pair, trade_type=TradeType. BUY, # It's always a buy during cointracking import amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type == 'Deposit' or row_type == 'Withdrawal': category = deserialize_asset_movement_category(row_type.lower()) if category == AssetMovementCategory.DEPOSIT: amount = deserialize_asset_amount(csv_row['Buy']) asset = Asset(csv_row['Cur.Buy']) else: amount = deserialize_asset_amount_force_positive( csv_row['Sell']) asset = Asset(csv_row['Cur.Sell']) asset_movement = AssetMovement( location=location, category=category, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_currency, link='', ) self.db.add_asset_movements([asset_movement]) else: raise UnsupportedCointrackingEntry( f'Unknown entrype type "{row_type}" encountered during cointracking ' f'data import. Ignoring entry', )
def query_grant_history_period( self, grant_id: int, grant_created_on: Optional[Timestamp], from_timestamp: Timestamp, to_timestamp: Timestamp, ) -> Tuple[List[LedgerAction], Optional[Timestamp]]: transactions: List[Dict[str, Any]] = [] if grant_created_on is None: query_str = ( f'https://gitcoin.co/api/v0.1/grants/contributions_rec_report/' f'?id={grant_id}&from_timestamp=2017-09-25&to_timestamp=2017-09-25' ) result = self._single_grant_api_query(query_str) try: grant_created_on = deserialize_timestamp_from_date( date=result['metadata']['created_on'], formatstr='%Y-%m-%dT%H:%M:%S', location='Gitcoin API', skip_milliseconds=True, ) from_timestamp = max(grant_created_on, from_timestamp) grant_name = result['metadata']['grant_name'] self.db_ledger.set_gitcoin_grant_metadata( grant_id=grant_id, name=grant_name, created_on=grant_created_on, ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' log.error( f'Unexpected data encountered during deserialization of gitcoin api ' f'query: {result}. Error was: {msg}', ) # continue with the given from_timestamp step_to_timestamp = min(from_timestamp + MONTH_IN_SECONDS, to_timestamp) while from_timestamp != step_to_timestamp: transactions.extend( self.query_grant_history_period30d( grant_id=grant_id, from_ts=from_timestamp, to_ts=Timestamp(step_to_timestamp), ), ) from_timestamp = Timestamp(step_to_timestamp) step_to_timestamp = min(step_to_timestamp + MONTH_IN_SECONDS, to_timestamp) # Check if any of the clr_payouts are in the range if self.clr_payouts: for payout in self.clr_payouts: timestamp = deserialize_timestamp_from_date( date=payout['timestamp'], formatstr='%Y-%m-%dT%H:%M:%S', location='Gitcoin API', skip_milliseconds=True, ) if from_timestamp <= timestamp <= to_timestamp: round_num = payout.pop('round') payout['clr_round'] = round_num transactions.append(payout) actions = [] for transaction in transactions: try: action = _deserialize_transaction(grant_id=grant_id, rawtx=transaction) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.db.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a gitcoin ' 'api query. Check logs for details.', ) log.error( f'Unexpected data encountered during deserialization of gitcoin api ' f'query: {transaction}. Error was: {msg}', ) continue except UnknownAsset as e: self.db.msg_aggregator.add_warning( f'Found unknown asset {str(e)} in a gitcoin api event transaction. ' 'Ignoring it.', ) continue except ZeroGitcoinAmount: log.warning( f'Found gitcoin event with 0 amount for grant {grant_id}. Ignoring' ) continue actions.append(action) return actions, grant_created_on