def query_income_loss_expense( self, start_ts: Timestamp, end_ts: Timestamp, only_cache: bool, ) -> List[LedgerAction]: """Queries the local DB and the exchange for the income/loss/expense history of the user If only_cache is true only what is already cached in the DB is returned without an actual exchange query. """ db = DBLedgerActions(self.db, self.db.msg_aggregator) filter_query = LedgerActionsFilterQuery.make( from_ts=start_ts, to_ts=end_ts, location=self.location, ) # has_premium True is fine here since the result of this is not user facing atm ledger_actions = db.get_ledger_actions(filter_query=filter_query, has_premium=True) if only_cache: return ledger_actions ranges = DBQueryRanges(self.db) location_string = f'{str(self.location)}_ledger_actions_{self.name}' ranges_to_query = ranges.get_location_query_ranges( location_string=location_string, start_ts=start_ts, end_ts=end_ts, ) new_ledger_actions = [] for query_start_ts, query_end_ts in ranges_to_query: new_ledger_actions.extend( self.query_online_income_loss_expense( start_ts=query_start_ts, end_ts=query_end_ts, )) if new_ledger_actions != []: db.add_ledger_actions(new_ledger_actions) ranges.update_used_query_range( location_string=location_string, start_ts=start_ts, end_ts=end_ts, ranges_to_query=ranges_to_query, ) ledger_actions.extend(new_ledger_actions) return ledger_actions
def test_store_same_tx_hash_in_db(database): """Test that if somehow during addition a duplicate is added, it's ignored and only 1 ends up in the db""" action1 = LedgerAction( identifier=1, timestamp=Timestamp(1624791600), action_type=LedgerActionType.DONATION_RECEIVED, location=Location.GITCOIN, amount=FVal('0.0004789924016679019628604417823'), asset=A_ETH, rate=FVal('1983.33'), rate_asset=A_USD, link= '0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1', notes='Gitcoin grant 149 event', extra_data=GitcoinEventData( tx_id= '0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1', grant_id=149, clr_round=None, tx_type=GitcoinEventTxType.ETHEREUM, ), ) action2 = LedgerAction( identifier=2, timestamp=Timestamp(1634791600), action_type=LedgerActionType.DONATION_RECEIVED, location=Location.GITCOIN, amount=FVal('0.789924016679019628604417823'), asset=A_ETH, rate=FVal('1913.33'), rate_asset=A_USD, link= '0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1', notes='Gitcoin grant 149 event', extra_data=GitcoinEventData( tx_id= '0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1', grant_id=149, clr_round=None, tx_type=GitcoinEventTxType.ETHEREUM, ), ) action3 = LedgerAction( identifier=2, timestamp=Timestamp(1654791600), action_type=LedgerActionType.DONATION_RECEIVED, location=Location.GITCOIN, amount=FVal('2445533521905078832065264'), asset=A_ETH, rate=FVal('1973.33'), rate_asset=A_USD, link= 'sync-tx:5612f84bc20cda25b911af39b792c973bdd5916b3b6868db2420b5dafd705a90', notes='Gitcoin grant 149 event', extra_data=GitcoinEventData( tx_id= '5612f84bc20cda25b911af39b792c973bdd5916b3b6868db2420b5dafd705a90', grant_id=149, clr_round=None, tx_type=GitcoinEventTxType.ZKSYNC, ), ) dbledger = DBLedgerActions(database, database.msg_aggregator) dbledger.add_ledger_actions([action1, action2, action3]) stored_actions = dbledger.get_ledger_actions( filter_query=LedgerActionsFilterQuery.make(location=Location.GITCOIN), has_premium=True, ) assert stored_actions == [action1, action3] errors = database.msg_aggregator.consume_errors() warnings = database.msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 1 assert 'Did not add ledger action to DB' in warnings[0]
class GitcoinDataImporter(): def __init__(self, db: DBHandler) -> None: self.db = db self.db_ledger = DBLedgerActions(self.db, self.db.msg_aggregator) self.grantid_re = re.compile(r'/grants/(\d+)/.*') def _consume_grant_entry(self, entry: Dict[str, Any]) -> Optional[LedgerAction]: """ Consumes a grant entry from the CSV and turns it into a LedgerAction May raise: - DeserializationError - KeyError - UnknownAsset """ if entry['Type'] != 'grant': return None timestamp = deserialize_timestamp_from_date( date=entry['date'], formatstr='%Y-%m-%dT%H:%M:%S', location='Gitcoin CSV', skip_milliseconds=True, ) usd_value = deserialize_asset_amount(entry['Value In USD']) asset = get_asset_by_symbol(entry['token_name']) if asset is None: raise UnknownAsset(entry['token_name']) token_amount = deserialize_asset_amount(entry['token_value']) if token_amount == ZERO: # try to make up for https://github.com/gitcoinco/web/issues/9213 price = query_usd_price_zero_if_error( asset=asset, time=timestamp, location=f'Gitcoin CSV entry {entry["txid"]}', msg_aggregator=self.db.msg_aggregator, ) if price == ZERO: self.db.msg_aggregator.add_warning( f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} ' f'due to amount being zero and inability to find price. Skipping.', ) return None # calculate the amount from price and value token_amount = usd_value / price # type: ignore match = self.grantid_re.search(entry['url']) if match is None: self.db.msg_aggregator.add_warning( f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} ' f'due to inability to read grant id. Skipping.', ) return None grant_id = int(match.group(1)) rate = Price(usd_value / token_amount) raw_txid = entry['txid'] tx_type, tx_id = process_gitcoin_txid(key='txid', entry=entry) return LedgerAction( identifier=1, # whatever does not go in the DB timestamp=timestamp, action_type=LedgerActionType.DONATION_RECEIVED, location=Location.GITCOIN, amount=token_amount, asset=asset, rate=rate, rate_asset=A_USD, # let's use the rate gitcoin calculated link=raw_txid, notes=f'Gitcoin grant {grant_id} event', extra_data=GitcoinEventData( tx_id=tx_id, grant_id=grant_id, clr_round=None, # can't get round from CSV tx_type=tx_type, ), ) def import_gitcoin_csv(self, filepath: Path) -> None: with open(filepath, 'r', encoding='utf-8-sig') as csvfile: data = csv.DictReader(csvfile, delimiter=',', quotechar='"') actions = [] for row in data: try: action = self._consume_grant_entry(row) except UnknownAsset as e: self.db.msg_aggregator.add_warning( f'During gitcoin grant CSV processing found asset {e.asset_name} ' f'that cant be matched to a single known asset. Skipping entry.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.db.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a gitcoin CSV ' 'entry. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of a gitcoin ' f'CSV entry: {row} . Error was: {msg}', ) continue if action: actions.append(action) db_actions = self.db_ledger.get_ledger_actions( filter_query=LedgerActionsFilterQuery.make( location=Location.GITCOIN), has_premium=True, ) existing_txids = [x.link for x in db_actions] self.db_ledger.add_ledger_actions( [x for x in actions if x.link not in existing_txids])
class GitcoinAPI(): def __init__(self, db: DBHandler) -> None: self.db = db self.db_ledger = DBLedgerActions(self.db, self.db.msg_aggregator) self.session = requests.session() self.clr_payouts: Optional[List[Dict[str, Any]]] = None def _single_grant_api_query(self, query_str: str) -> Dict[str, Any]: backoff = 1 backoff_limit = 33 while backoff < backoff_limit: log.debug(f'Querying gitcoin: {query_str}') try: response = self.session.get(query_str, timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: if 'Max retries exceeded with url' in str(e): log.debug( f'Got max retries exceeded from gitcoin. Will ' f'backoff for {backoff} seconds.', ) gevent.sleep(backoff) backoff = backoff * 2 if backoff >= backoff_limit: raise RemoteError( 'Getting gitcoin error even ' 'after we incrementally backed off', ) from e continue raise RemoteError( f'Gitcoin API request failed due to {str(e)}') from e if response.status_code != 200: raise RemoteError( f'Gitcoin API request {response.url} failed ' f'with HTTP status code {response.status_code} and text ' f'{response.text}', ) try: json_ret = jsonloads_dict(response.text) except JSONDecodeError as e: raise RemoteError( f'Gitcoin API request {response.url} returned invalid ' f'JSON response: {response.text}', ) from e if 'error' in json_ret: raise RemoteError( f'Gitcoin API request {response.url} returned an error: {json_ret["error"]}', ) break # success return json_ret def get_history_from_db( self, grant_id: Optional[int], from_ts: Optional[Timestamp] = None, to_ts: Optional[Timestamp] = None, ) -> Dict[int, Dict[str, Any]]: grantid_to_metadata = self.db_ledger.get_gitcoin_grant_metadata( grant_id) grantid_to_events = defaultdict(list) events = self.db_ledger.get_gitcoin_grant_events( grant_id=grant_id, from_ts=from_ts, to_ts=to_ts, ) for event in events: grantid_to_events[event.extra_data.grant_id].append( event.serialize_for_gitcoin()) # type: ignore # noqa: E501 result = {} for grantid, serialized_events in grantid_to_events.items(): metadata = grantid_to_metadata.get(grantid) result[grantid] = { 'events': serialized_events, 'name': metadata.name if metadata else None, 'created_on': metadata.created_on if metadata else None, } return result def query_grant_history( self, grant_id: Optional[int], from_ts: Optional[Timestamp] = None, to_ts: Optional[Timestamp] = None, only_cache: bool = False, ) -> Dict[int, Dict[str, Any]]: """May raise: - RemotError if there is an error querying the gitcoin API - InputError if only_cache is False and grant_id is missing """ if only_cache: return self.get_history_from_db( grant_id=grant_id, from_ts=from_ts, to_ts=to_ts, ) if grant_id is None: raise InputError( 'Attempted to query gitcoin events from the api without specifying a grant id', ) entry_name = f'{GITCOIN_GRANTS_PREFIX}_{grant_id}' dbranges = DBQueryRanges(self.db) from_timestamp = GITCOIN_START_TS if from_ts is None else from_ts to_timestamp = ts_now() if to_ts is None else to_ts ranges = dbranges.get_location_query_ranges( location_string=entry_name, start_ts=from_timestamp, end_ts=to_timestamp, ) grant_created_on: Optional[Timestamp] = None for period_range in ranges: actions, grant_created_on = self.query_grant_history_period( grant_id=grant_id, grant_created_on=grant_created_on, from_timestamp=period_range[0], to_timestamp=period_range[1], ) self.db_ledger.add_ledger_actions(actions) dbranges.update_used_query_range( location_string=entry_name, start_ts=from_timestamp, end_ts=to_timestamp, ranges_to_query=ranges, ) return self.get_history_from_db( grant_id=grant_id, from_ts=from_ts, to_ts=to_ts, ) def query_grant_history_period( self, grant_id: int, grant_created_on: Optional[Timestamp], from_timestamp: Timestamp, to_timestamp: Timestamp, ) -> Tuple[List[LedgerAction], Optional[Timestamp]]: transactions: List[Dict[str, Any]] = [] if grant_created_on is None: query_str = ( f'https://gitcoin.co/api/v0.1/grants/contributions_rec_report/' f'?id={grant_id}&from_timestamp=2017-09-25&to_timestamp=2017-09-25' ) result = self._single_grant_api_query(query_str) try: grant_created_on = deserialize_timestamp_from_date( date=result['metadata']['created_on'], formatstr='%Y-%m-%dT%H:%M:%S', location='Gitcoin API', skip_milliseconds=True, ) from_timestamp = max(grant_created_on, from_timestamp) grant_name = result['metadata']['grant_name'] self.db_ledger.set_gitcoin_grant_metadata( grant_id=grant_id, name=grant_name, created_on=grant_created_on, ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' log.error( f'Unexpected data encountered during deserialization of gitcoin api ' f'query: {result}. Error was: {msg}', ) # continue with the given from_timestamp step_to_timestamp = min(from_timestamp + MONTH_IN_SECONDS, to_timestamp) while from_timestamp != step_to_timestamp: transactions.extend( self.query_grant_history_period30d( grant_id=grant_id, from_ts=from_timestamp, to_ts=Timestamp(step_to_timestamp), ), ) from_timestamp = Timestamp(step_to_timestamp) step_to_timestamp = min(step_to_timestamp + MONTH_IN_SECONDS, to_timestamp) # Check if any of the clr_payouts are in the range if self.clr_payouts: for payout in self.clr_payouts: timestamp = deserialize_timestamp_from_date( date=payout['timestamp'], formatstr='%Y-%m-%dT%H:%M:%S', location='Gitcoin API', skip_milliseconds=True, ) if from_timestamp <= timestamp <= to_timestamp: round_num = payout.pop('round') payout['clr_round'] = round_num transactions.append(payout) actions = [] for transaction in transactions: try: action = _deserialize_transaction(grant_id=grant_id, rawtx=transaction) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.db.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a gitcoin ' 'api query. Check logs for details.', ) log.error( f'Unexpected data encountered during deserialization of gitcoin api ' f'query: {transaction}. Error was: {msg}', ) continue except UnknownAsset as e: self.db.msg_aggregator.add_warning( f'Found unknown asset {str(e)} in a gitcoin api event transaction. ' 'Ignoring it.', ) continue except ZeroGitcoinAmount: log.warning( f'Found gitcoin event with 0 amount for grant {grant_id}. Ignoring' ) continue actions.append(action) return actions, grant_created_on def query_grant_history_period30d( self, grant_id: int, from_ts: Timestamp, to_ts: Timestamp, ) -> List[Dict[str, Any]]: transactions = [] from_date = timestamp_to_date(from_ts, formatstr='%Y-%m-%d') to_date = timestamp_to_date(to_ts, formatstr='%Y-%m-%d') page = 1 while True: query_str = ( f'https://gitcoin.co/api/v0.1/grants/contributions_rec_report/' f'?id={grant_id}&from_timestamp={from_date}&to_timestamp={to_date}' f'&page={page}&format=json') result = self._single_grant_api_query(query_str) transactions.extend(result['transactions']) if self.clr_payouts is None: self.clr_payouts = result.get('clr_payouts', []) if result['metadata']['has_next'] is False: break # else next page page += 1 return transactions