def query_income_loss_expense(
self,
start_ts: Timestamp,
end_ts: Timestamp,
only_cache: bool,
) -> List[LedgerAction]:
"""Queries the local DB and the exchange for the income/loss/expense history of the user
If only_cache is true only what is already cached in the DB is returned without
an actual exchange query.
"""
db = DBLedgerActions(self.db, self.db.msg_aggregator)
filter_query = LedgerActionsFilterQuery.make(
from_ts=start_ts,
to_ts=end_ts,
location=self.location,
)
# has_premium True is fine here since the result of this is not user facing atm
ledger_actions = db.get_ledger_actions(filter_query=filter_query,
has_premium=True)
if only_cache:
return ledger_actions
ranges = DBQueryRanges(self.db)
location_string = f'{str(self.location)}_ledger_actions_{self.name}'
ranges_to_query = ranges.get_location_query_ranges(
location_string=location_string,
start_ts=start_ts,
end_ts=end_ts,
)
new_ledger_actions = []
for query_start_ts, query_end_ts in ranges_to_query:
new_ledger_actions.extend(
self.query_online_income_loss_expense(
start_ts=query_start_ts,
end_ts=query_end_ts,
))
if new_ledger_actions != []:
db.add_ledger_actions(new_ledger_actions)
ranges.update_used_query_range(
location_string=location_string,
start_ts=start_ts,
end_ts=end_ts,
ranges_to_query=ranges_to_query,
)
ledger_actions.extend(new_ledger_actions)
return ledger_actions
def test_store_same_tx_hash_in_db(database):
"""Test that if somehow during addition a duplicate is added,
it's ignored and only 1 ends up in the db"""
action1 = LedgerAction(
identifier=1,
timestamp=Timestamp(1624791600),
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=FVal('0.0004789924016679019628604417823'),
asset=A_ETH,
rate=FVal('1983.33'),
rate_asset=A_USD,
link=
'0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1',
notes='Gitcoin grant 149 event',
extra_data=GitcoinEventData(
tx_id=
'0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1',
grant_id=149,
clr_round=None,
tx_type=GitcoinEventTxType.ETHEREUM,
),
)
action2 = LedgerAction(
identifier=2,
timestamp=Timestamp(1634791600),
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=FVal('0.789924016679019628604417823'),
asset=A_ETH,
rate=FVal('1913.33'),
rate_asset=A_USD,
link=
'0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1',
notes='Gitcoin grant 149 event',
extra_data=GitcoinEventData(
tx_id=
'0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1',
grant_id=149,
clr_round=None,
tx_type=GitcoinEventTxType.ETHEREUM,
),
)
action3 = LedgerAction(
identifier=2,
timestamp=Timestamp(1654791600),
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=FVal('2445533521905078832065264'),
asset=A_ETH,
rate=FVal('1973.33'),
rate_asset=A_USD,
link=
'sync-tx:5612f84bc20cda25b911af39b792c973bdd5916b3b6868db2420b5dafd705a90',
notes='Gitcoin grant 149 event',
extra_data=GitcoinEventData(
tx_id=
'5612f84bc20cda25b911af39b792c973bdd5916b3b6868db2420b5dafd705a90',
grant_id=149,
clr_round=None,
tx_type=GitcoinEventTxType.ZKSYNC,
),
)
dbledger = DBLedgerActions(database, database.msg_aggregator)
dbledger.add_ledger_actions([action1, action2, action3])
stored_actions = dbledger.get_ledger_actions(
filter_query=LedgerActionsFilterQuery.make(location=Location.GITCOIN),
has_premium=True,
)
assert stored_actions == [action1, action3]
errors = database.msg_aggregator.consume_errors()
warnings = database.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 1
assert 'Did not add ledger action to DB' in warnings[0]
class GitcoinDataImporter():
def __init__(self, db: DBHandler) -> None:
self.db = db
self.db_ledger = DBLedgerActions(self.db, self.db.msg_aggregator)
self.grantid_re = re.compile(r'/grants/(\d+)/.*')
def _consume_grant_entry(self, entry: Dict[str,
Any]) -> Optional[LedgerAction]:
"""
Consumes a grant entry from the CSV and turns it into a LedgerAction
May raise:
- DeserializationError
- KeyError
- UnknownAsset
"""
if entry['Type'] != 'grant':
return None
timestamp = deserialize_timestamp_from_date(
date=entry['date'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin CSV',
skip_milliseconds=True,
)
usd_value = deserialize_asset_amount(entry['Value In USD'])
asset = get_asset_by_symbol(entry['token_name'])
if asset is None:
raise UnknownAsset(entry['token_name'])
token_amount = deserialize_asset_amount(entry['token_value'])
if token_amount == ZERO: # try to make up for https://github.com/gitcoinco/web/issues/9213
price = query_usd_price_zero_if_error(
asset=asset,
time=timestamp,
location=f'Gitcoin CSV entry {entry["txid"]}',
msg_aggregator=self.db.msg_aggregator,
)
if price == ZERO:
self.db.msg_aggregator.add_warning(
f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} '
f'due to amount being zero and inability to find price. Skipping.',
)
return None
# calculate the amount from price and value
token_amount = usd_value / price # type: ignore
match = self.grantid_re.search(entry['url'])
if match is None:
self.db.msg_aggregator.add_warning(
f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} '
f'due to inability to read grant id. Skipping.', )
return None
grant_id = int(match.group(1))
rate = Price(usd_value / token_amount)
raw_txid = entry['txid']
tx_type, tx_id = process_gitcoin_txid(key='txid', entry=entry)
return LedgerAction(
identifier=1, # whatever does not go in the DB
timestamp=timestamp,
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=token_amount,
asset=asset,
rate=rate,
rate_asset=A_USD, # let's use the rate gitcoin calculated
link=raw_txid,
notes=f'Gitcoin grant {grant_id} event',
extra_data=GitcoinEventData(
tx_id=tx_id,
grant_id=grant_id,
clr_round=None, # can't get round from CSV
tx_type=tx_type,
),
)
def import_gitcoin_csv(self, filepath: Path) -> None:
with open(filepath, 'r', encoding='utf-8-sig') as csvfile:
data = csv.DictReader(csvfile, delimiter=',', quotechar='"')
actions = []
for row in data:
try:
action = self._consume_grant_entry(row)
except UnknownAsset as e:
self.db.msg_aggregator.add_warning(
f'During gitcoin grant CSV processing found asset {e.asset_name} '
f'that cant be matched to a single known asset. Skipping entry.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.db.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a gitcoin CSV '
'entry. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of a gitcoin '
f'CSV entry: {row} . Error was: {msg}', )
continue
if action:
actions.append(action)
db_actions = self.db_ledger.get_ledger_actions(
filter_query=LedgerActionsFilterQuery.make(
location=Location.GITCOIN),
has_premium=True,
)
existing_txids = [x.link for x in db_actions]
self.db_ledger.add_ledger_actions(
[x for x in actions if x.link not in existing_txids])
class GitcoinAPI():
def __init__(self, db: DBHandler) -> None:
self.db = db
self.db_ledger = DBLedgerActions(self.db, self.db.msg_aggregator)
self.session = requests.session()
self.clr_payouts: Optional[List[Dict[str, Any]]] = None
def _single_grant_api_query(self, query_str: str) -> Dict[str, Any]:
backoff = 1
backoff_limit = 33
while backoff < backoff_limit:
log.debug(f'Querying gitcoin: {query_str}')
try:
response = self.session.get(query_str,
timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
if 'Max retries exceeded with url' in str(e):
log.debug(
f'Got max retries exceeded from gitcoin. Will '
f'backoff for {backoff} seconds.', )
gevent.sleep(backoff)
backoff = backoff * 2
if backoff >= backoff_limit:
raise RemoteError(
'Getting gitcoin error even '
'after we incrementally backed off', ) from e
continue
raise RemoteError(
f'Gitcoin API request failed due to {str(e)}') from e
if response.status_code != 200:
raise RemoteError(
f'Gitcoin API request {response.url} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text}', )
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Gitcoin API request {response.url} returned invalid '
f'JSON response: {response.text}', ) from e
if 'error' in json_ret:
raise RemoteError(
f'Gitcoin API request {response.url} returned an error: {json_ret["error"]}',
)
break # success
return json_ret
def get_history_from_db(
self,
grant_id: Optional[int],
from_ts: Optional[Timestamp] = None,
to_ts: Optional[Timestamp] = None,
) -> Dict[int, Dict[str, Any]]:
grantid_to_metadata = self.db_ledger.get_gitcoin_grant_metadata(
grant_id)
grantid_to_events = defaultdict(list)
events = self.db_ledger.get_gitcoin_grant_events(
grant_id=grant_id,
from_ts=from_ts,
to_ts=to_ts,
)
for event in events:
grantid_to_events[event.extra_data.grant_id].append(
event.serialize_for_gitcoin()) # type: ignore # noqa: E501
result = {}
for grantid, serialized_events in grantid_to_events.items():
metadata = grantid_to_metadata.get(grantid)
result[grantid] = {
'events': serialized_events,
'name': metadata.name if metadata else None,
'created_on': metadata.created_on if metadata else None,
}
return result
def query_grant_history(
self,
grant_id: Optional[int],
from_ts: Optional[Timestamp] = None,
to_ts: Optional[Timestamp] = None,
only_cache: bool = False,
) -> Dict[int, Dict[str, Any]]:
"""May raise:
- RemotError if there is an error querying the gitcoin API
- InputError if only_cache is False and grant_id is missing
"""
if only_cache:
return self.get_history_from_db(
grant_id=grant_id,
from_ts=from_ts,
to_ts=to_ts,
)
if grant_id is None:
raise InputError(
'Attempted to query gitcoin events from the api without specifying a grant id',
)
entry_name = f'{GITCOIN_GRANTS_PREFIX}_{grant_id}'
dbranges = DBQueryRanges(self.db)
from_timestamp = GITCOIN_START_TS if from_ts is None else from_ts
to_timestamp = ts_now() if to_ts is None else to_ts
ranges = dbranges.get_location_query_ranges(
location_string=entry_name,
start_ts=from_timestamp,
end_ts=to_timestamp,
)
grant_created_on: Optional[Timestamp] = None
for period_range in ranges:
actions, grant_created_on = self.query_grant_history_period(
grant_id=grant_id,
grant_created_on=grant_created_on,
from_timestamp=period_range[0],
to_timestamp=period_range[1],
)
self.db_ledger.add_ledger_actions(actions)
dbranges.update_used_query_range(
location_string=entry_name,
start_ts=from_timestamp,
end_ts=to_timestamp,
ranges_to_query=ranges,
)
return self.get_history_from_db(
grant_id=grant_id,
from_ts=from_ts,
to_ts=to_ts,
)
def query_grant_history_period(
self,
grant_id: int,
grant_created_on: Optional[Timestamp],
from_timestamp: Timestamp,
to_timestamp: Timestamp,
) -> Tuple[List[LedgerAction], Optional[Timestamp]]:
transactions: List[Dict[str, Any]] = []
if grant_created_on is None:
query_str = (
f'https://gitcoin.co/api/v0.1/grants/contributions_rec_report/'
f'?id={grant_id}&from_timestamp=2017-09-25&to_timestamp=2017-09-25'
)
result = self._single_grant_api_query(query_str)
try:
grant_created_on = deserialize_timestamp_from_date(
date=result['metadata']['created_on'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin API',
skip_milliseconds=True,
)
from_timestamp = max(grant_created_on, from_timestamp)
grant_name = result['metadata']['grant_name']
self.db_ledger.set_gitcoin_grant_metadata(
grant_id=grant_id,
name=grant_name,
created_on=grant_created_on,
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
log.error(
f'Unexpected data encountered during deserialization of gitcoin api '
f'query: {result}. Error was: {msg}', )
# continue with the given from_timestamp
step_to_timestamp = min(from_timestamp + MONTH_IN_SECONDS,
to_timestamp)
while from_timestamp != step_to_timestamp:
transactions.extend(
self.query_grant_history_period30d(
grant_id=grant_id,
from_ts=from_timestamp,
to_ts=Timestamp(step_to_timestamp),
), )
from_timestamp = Timestamp(step_to_timestamp)
step_to_timestamp = min(step_to_timestamp + MONTH_IN_SECONDS,
to_timestamp)
# Check if any of the clr_payouts are in the range
if self.clr_payouts:
for payout in self.clr_payouts:
timestamp = deserialize_timestamp_from_date(
date=payout['timestamp'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin API',
skip_milliseconds=True,
)
if from_timestamp <= timestamp <= to_timestamp:
round_num = payout.pop('round')
payout['clr_round'] = round_num
transactions.append(payout)
actions = []
for transaction in transactions:
try:
action = _deserialize_transaction(grant_id=grant_id,
rawtx=transaction)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.db.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a gitcoin '
'api query. Check logs for details.', )
log.error(
f'Unexpected data encountered during deserialization of gitcoin api '
f'query: {transaction}. Error was: {msg}', )
continue
except UnknownAsset as e:
self.db.msg_aggregator.add_warning(
f'Found unknown asset {str(e)} in a gitcoin api event transaction. '
'Ignoring it.', )
continue
except ZeroGitcoinAmount:
log.warning(
f'Found gitcoin event with 0 amount for grant {grant_id}. Ignoring'
)
continue
actions.append(action)
return actions, grant_created_on
def query_grant_history_period30d(
self,
grant_id: int,
from_ts: Timestamp,
to_ts: Timestamp,
) -> List[Dict[str, Any]]:
transactions = []
from_date = timestamp_to_date(from_ts, formatstr='%Y-%m-%d')
to_date = timestamp_to_date(to_ts, formatstr='%Y-%m-%d')
page = 1
while True:
query_str = (
f'https://gitcoin.co/api/v0.1/grants/contributions_rec_report/'
f'?id={grant_id}&from_timestamp={from_date}&to_timestamp={to_date}'
f'&page={page}&format=json')
result = self._single_grant_api_query(query_str)
transactions.extend(result['transactions'])
if self.clr_payouts is None:
self.clr_payouts = result.get('clr_payouts', [])
if result['metadata']['has_next'] is False:
break
# else next page
page += 1
return transactions
