def _refetch_repos_for_account(rl_prefix, rl_key, participant, account):
sess = account.get_auth_session()
logger.debug(
"Refetching profile data for participant ~%s from %s account %s" %
(participant.id, account.platform, account.user_id))
try:
account = account.refresh_user_info()
except (UnableToRefreshAccount, UserNotFound) as e:
logger.debug("The refetch failed: %s" % e)
sleep(1)
logger.debug(
"Refetching repository data for participant ~%s from %s account %s" %
(participant.id, account.platform, account.user_id))
start_time = utcnow()
try:
with website.db.get_cursor() as cursor:
next_page = None
for i in range(10):
r = account.platform_data.get_repos(account,
page_url=next_page,
sess=sess)
upsert_repos(cursor, r[0], participant, utcnow())
next_page = r[2].get('next')
if not next_page:
break
sleep(1)
deleted_count = cursor.one(
"""
WITH deleted AS (
DELETE FROM repositories
WHERE participant = %s
AND platform = %s
AND info_fetched_at < %s
RETURNING id
)
SELECT count(*) FROM deleted
""", (participant.id, account.platform, start_time))
event_type = 'fetch_repos:%s' % account.id
payload = dict(partial_list=bool(next_page),
deleted_count=deleted_count)
participant.add_event(cursor, event_type, payload)
cursor.run(
"""
DELETE FROM events
WHERE participant = %s
AND type = %s
AND (NOT payload ? 'deleted_count' OR payload->'deleted_count' = '0')
AND ts < (current_timestamp - interval '6 days')
""", (participant.id, event_type))
except (InvalidGrantError, TokenExpiredError) as e:
logger.debug("The refetch failed: %s" % e)
return
# The update was successful, clean up the rate_limiting table
website.db.run("DELETE FROM rate_limiting WHERE key = %s",
(rl_prefix + ':' + rl_key, ))
def test_dispute_callback_lost(self, save, get_payin, get_dispute):
self.make_participant(
'LiberapayOrg',
kind='organization',
balance=EUR('100.00'),
mangopay_user_id='0',
mangopay_wallet_id='0',
)
save.side_effect = fake_transfer
e_id = self.make_exchange('mango-cc', EUR('16'), EUR('1'), self.janet)
dispute = Dispute()
dispute.Id = '-1'
dispute.CreationDate = utcnow()
dispute.DisputedFunds = Money(1700, 'EUR')
dispute.DisputeType = 'CONTESTABLE'
dispute.InitialTransactionType = 'PAYIN'
get_dispute.return_value = dispute
payin = PayIn(tag=str(e_id))
get_payin.return_value = payin
# Transfer some of the money to homer
self.janet.set_tip_to(self.homer, EUR('3.68'))
Payday.start().run()
# Withdraw some of the money
self.make_exchange('mango-ba', EUR('-2.68'), 0, self.homer)
# Add a bit of money that will remain undisputed, to test bundle swapping
self.make_exchange('mango-cc', EUR('0.32'), 0, self.janet)
self.make_exchange('mango-cc', EUR('0.55'), 0, self.homer)
# Call back
self.db.self_check()
for status in ('CREATED', 'CLOSED'):
dispute.Status = status
if status == 'CLOSED':
dispute.ResultCode = 'LOST'
qs = "EventType=DISPUTE_" + status + "&RessourceId=123456790"
r = self.callback(qs, raise_immediately=True)
assert r.code == 200, r.text
self.db.self_check()
# Check final state
balances = dict(
self.db.all("SELECT username, balance FROM participants"))
assert balances == {
'_chargebacks_': EUR('16.00'),
'david': 0,
'homer': 0,
'janet': 0,
'LiberapayOrg': EUR('98.19'),
}
debts = dict(((r[0], r[1]), r[2]) for r in self.db.all("""
SELECT p_debtor.username AS debtor, p_creditor.username AS creditor, sum(d.amount)
FROM debts d
JOIN participants p_debtor ON p_debtor.id = d.debtor
JOIN participants p_creditor ON p_creditor.id = d.creditor
WHERE d.status = 'due'
GROUP BY p_debtor.username, p_creditor.username
"""))
assert debts == {
('janet', 'LiberapayOrg'): EUR('1.00'),
('janet', 'homer'): EUR('3.36'),
('homer', 'LiberapayOrg'): EUR('1.81'),
}
def refetch_repos():
with website.db.get_cursor() as cursor:
repo = cursor.one("""
SELECT r.participant, r.platform
FROM repositories r
WHERE r.info_fetched_at < now() - interval '6 days'
AND r.participant IS NOT NULL
ORDER BY r.info_fetched_at ASC
LIMIT 1
""")
if not repo:
return
participant = Participant.from_id(repo.participant)
account = participant.get_account_elsewhere(repo.platform)
sess = account.get_auth_session()
start_time = utcnow()
logger.debug(
"Refetching repository data for participant ~%s from %s account %s"
% (participant.id, account.platform, account.user_id))
next_page = None
for i in range(10):
r = account.platform_data.get_repos(account,
page_url=next_page,
sess=sess)
upsert_repos(cursor, r[0], participant, utcnow())
next_page = r[2].get('next')
if not next_page:
break
sleep(1)
deleted_count = cursor.one(
"""
WITH deleted AS (
DELETE FROM repositories
WHERE participant = %s
AND platform = %s
AND info_fetched_at < %s
RETURNING id
)
SELECT count(*) FROM deleted
""", (participant.id, account.platform, start_time))
event_type = 'fetch_repos:%s' % account.id
payload = dict(partial_list=bool(next_page),
deleted_count=deleted_count)
participant.add_event(cursor, event_type, payload)
def test_dispute_callback_lost(self, save, get_payin, get_dispute):
self.make_participant(
'LiberapayOrg', kind='organization', balance=D('100.00'),
mangopay_user_id='0', mangopay_wallet_id='0',
)
save.side_effect = fake_transfer
e_id = self.make_exchange('mango-cc', D('16'), D('1'), self.janet)
dispute = Dispute()
dispute.Id = '-1'
dispute.CreationDate = utcnow()
dispute.DisputedFunds = Money(1700, 'EUR')
dispute.DisputeType = 'CONTESTABLE'
dispute.InitialTransactionType = 'PAYIN'
get_dispute.return_value = dispute
payin = PayIn(tag=str(e_id))
get_payin.return_value = payin
# Transfer some of the money to homer
self.janet.set_tip_to(self.homer, EUR('3.68'))
Payday.start().run()
# Withdraw some of the money
self.make_exchange('mango-ba', D('-2.68'), 0, self.homer)
# Add a bit of money that will remain undisputed, to test bundle swapping
self.make_exchange('mango-cc', D('0.32'), 0, self.janet)
self.make_exchange('mango-cc', D('0.55'), 0, self.homer)
# Call back
self.db.self_check()
for status in ('CREATED', 'CLOSED'):
dispute.Status = status
if status == 'CLOSED':
dispute.ResultCode = 'LOST'
qs = "EventType=DISPUTE_"+status+"&RessourceId=123456790"
r = self.callback(qs, raise_immediately=True)
assert r.code == 200, r.text
self.db.self_check()
# Check final state
balances = dict(self.db.all("SELECT username, balance FROM participants"))
assert balances == {
'_chargebacks_': D('16.00'),
'david': 0,
'homer': 0,
'janet': 0,
'LiberapayOrg': D('98.19'),
}
debts = dict(((r[0], r[1]), r[2]) for r in self.db.all("""
SELECT p_debtor.username AS debtor, p_creditor.username AS creditor, sum(d.amount)
FROM debts d
JOIN participants p_debtor ON p_debtor.id = d.debtor
JOIN participants p_creditor ON p_creditor.id = d.creditor
WHERE d.status = 'due'
GROUP BY p_debtor.username, p_creditor.username
"""))
assert debts == {
('janet', 'LiberapayOrg'): D('1.00'),
('janet', 'homer'): D('3.36'),
('homer', 'LiberapayOrg'): D('1.81'),
}
def refetch_repos():
with website.db.get_cursor() as cursor:
repo = cursor.one("""
SELECT r.participant, r.platform
FROM repositories r
WHERE r.info_fetched_at < now() - interval '6 days'
AND r.participant IS NOT NULL
ORDER BY r.info_fetched_at ASC
LIMIT 1
""")
if not repo:
return
participant = Participant.from_id(repo.participant)
account = participant.get_account_elsewhere(repo.platform)
sess = account.get_auth_session()
start_time = utcnow()
logger.debug(
"Refetching repository data for participant ~%s from %s account %s" %
(participant.id, account.platform, account.user_id)
)
next_page = None
for i in range(10):
r = account.platform_data.get_repos(account, page_url=next_page, sess=sess)
upsert_repos(cursor, r[0], participant, utcnow())
next_page = r[2].get('next')
if not next_page:
break
sleep(1)
deleted_count = cursor.one("""
WITH deleted AS (
DELETE FROM repositories
WHERE participant = %s
AND platform = %s
AND info_fetched_at < %s
RETURNING id
)
SELECT count(*) FROM deleted
""", (participant.id, account.platform, start_time))
event_type = 'fetch_repos:%s' % account.id
payload = dict(partial_list=bool(next_page), deleted_count=deleted_count)
participant.add_event(cursor, event_type, payload)
def test_dispute_callback_won(self, save, get_payin, get_dispute):
self.make_participant('LiberapayOrg', kind='organization')
save.side_effect = fake_transfer
e_id = self.make_exchange('mango-cc', EUR('16'), EUR('1'), self.janet)
dispute = Dispute()
dispute.Id = '-1'
dispute.CreationDate = utcnow()
dispute.DisputedFunds = Money(1700, 'EUR')
dispute.DisputeType = 'CONTESTABLE'
dispute.InitialTransactionType = 'PAYIN'
get_dispute.return_value = dispute
payin = PayIn(tag=str(e_id))
get_payin.return_value = payin
# Transfer some of the money to homer
self.janet.set_tip_to(self.homer, EUR('3.68'))
Payday.start().run()
# Withdraw some of the money
self.make_exchange('mango-ba', EUR('-2.68'), 0, self.homer)
# Add money that will remain undisputed, to test bundle swapping
self.make_exchange('mango-cc', EUR('2.69'), 0, self.janet)
# Call back
self.db.self_check()
for status in ('CREATED', 'CLOSED'):
dispute.Status = status
if status == 'CLOSED':
dispute.ResultCode = 'WON'
qs = "EventType=DISPUTE_" + status + "&RessourceId=123456790"
r = self.callback(qs)
assert r.code == 200, r.text
self.db.self_check()
# Check final state
disputed = self.db.all("SELECT * FROM cash_bundles WHERE disputed")
debts = self.db.all("SELECT * FROM debts")
assert not disputed
assert not debts
balances = dict(
self.db.all("SELECT username, balance FROM participants"))
assert balances == {
'david': 0,
'homer': EUR('1.00'),
'janet': EUR('15.01'),
'LiberapayOrg': 0,
}
def test_dispute_callback_won(self, save, get_payin, get_dispute):
self.make_participant('LiberapayOrg', kind='organization')
save.side_effect = fake_transfer
e_id = self.make_exchange('mango-cc', D('16'), D('1'), self.janet)
dispute = Dispute()
dispute.Id = '-1'
dispute.CreationDate = utcnow()
dispute.DisputedFunds = Money(1700, 'EUR')
dispute.DisputeType = 'CONTESTABLE'
dispute.InitialTransactionType = 'PAYIN'
get_dispute.return_value = dispute
payin = PayIn(tag=str(e_id))
get_payin.return_value = payin
# Transfer some of the money to homer
self.janet.set_tip_to(self.homer, EUR('3.68'))
Payday.start().run()
# Withdraw some of the money
self.make_exchange('mango-ba', D('-2.68'), 0, self.homer)
# Add money that will remain undisputed, to test bundle swapping
self.make_exchange('mango-cc', D('2.69'), 0, self.janet)
# Call back
self.db.self_check()
for status in ('CREATED', 'CLOSED'):
dispute.Status = status
if status == 'CLOSED':
dispute.ResultCode = 'WON'
qs = "EventType=DISPUTE_"+status+"&RessourceId=123456790"
r = self.callback(qs)
assert r.code == 200, r.text
self.db.self_check()
# Check final state
disputed = self.db.all("SELECT * FROM cash_bundles WHERE disputed")
debts = self.db.all("SELECT * FROM debts")
assert not disputed
assert not debts
balances = dict(self.db.all("SELECT username, balance FROM participants"))
assert balances == {
'david': 0,
'homer': D('1.00'),
'janet': D('15.01'),
'LiberapayOrg': 0,
}
def refetch_repos():
# Note: the rate_limiting table is used to avoid blocking on errors
rl_prefix = 'refetch_repos'
rl_cap, rl_period = RATE_LIMITS[rl_prefix]
repo = website.db.one(
"""
SELECT r.participant, r.platform
FROM repositories r
WHERE r.info_fetched_at < now() - interval '6 days'
AND r.participant IS NOT NULL
AND r.show_on_profile
AND check_rate_limit(%s || r.participant::text || ':' || r.platform, %s, %s)
ORDER BY r.info_fetched_at ASC
LIMIT 1
""", (rl_prefix + ':', rl_cap, rl_period))
if not repo:
return
rl_key = '%s:%s' % (repo.participant, repo.platform)
website.db.hit_rate_limit(rl_prefix, rl_key)
participant = Participant.from_id(repo.participant)
account = participant.get_account_elsewhere(repo.platform)
sess = account.get_auth_session()
logger.debug(
"Refetching profile data for participant ~%s from %s account %s" %
(participant.id, account.platform, account.user_id))
account = account.refresh_user_info()
sleep(1)
logger.debug(
"Refetching repository data for participant ~%s from %s account %s" %
(participant.id, account.platform, account.user_id))
start_time = utcnow()
with website.db.get_cursor() as cursor:
next_page = None
for i in range(10):
r = account.platform_data.get_repos(account,
page_url=next_page,
sess=sess)
upsert_repos(cursor, r[0], participant, utcnow())
next_page = r[2].get('next')
if not next_page:
break
sleep(1)
deleted_count = cursor.one(
"""
WITH deleted AS (
DELETE FROM repositories
WHERE participant = %s
AND platform = %s
AND info_fetched_at < %s
RETURNING id
)
SELECT count(*) FROM deleted
""", (participant.id, account.platform, start_time))
event_type = 'fetch_repos:%s' % account.id
payload = dict(partial_list=bool(next_page),
deleted_count=deleted_count)
participant.add_event(cursor, event_type, payload)
cursor.run(
"""
DELETE FROM events
WHERE participant = %s
AND type = %s
AND (NOT payload ? 'deleted_count' OR payload->'deleted_count' = '0')
AND ts < (current_timestamp - interval '6 days')
""", (participant.id, event_type))
# The update was successful, clean up the rate_limiting table
website.db.run("DELETE FROM rate_limiting WHERE key = %s",
(rl_prefix + ':' + rl_key, ))
def refetch_repos():
# Note: the rate_limiting table is used to avoid blocking on errors
rl_prefix = 'refetch_repos'
rl_cap, rl_period = RATE_LIMITS[rl_prefix]
repo = website.db.one("""
SELECT r.participant, r.platform
FROM repositories r
WHERE r.info_fetched_at < now() - interval '6 days'
AND r.participant IS NOT NULL
AND r.show_on_profile
AND check_rate_limit(%s || r.participant::text || ':' || r.platform, %s, %s)
ORDER BY r.info_fetched_at ASC
LIMIT 1
""", (rl_prefix + ':', rl_cap, rl_period))
if not repo:
return
rl_key = '%s:%s' % (repo.participant, repo.platform)
website.db.hit_rate_limit(rl_prefix, rl_key)
participant = Participant.from_id(repo.participant)
account = participant.get_account_elsewhere(repo.platform)
if not account:
return
sess = account.get_auth_session()
logger.debug(
"Refetching profile data for participant ~%s from %s account %s" %
(participant.id, account.platform, account.user_id)
)
try:
account = account.refresh_user_info()
except (UnableToRefreshAccount, UserNotFound) as e:
logger.debug("The refetch failed: %s" % e)
sleep(1)
logger.debug(
"Refetching repository data for participant ~%s from %s account %s" %
(participant.id, account.platform, account.user_id)
)
start_time = utcnow()
try:
with website.db.get_cursor() as cursor:
next_page = None
for i in range(10):
r = account.platform_data.get_repos(account, page_url=next_page, sess=sess)
upsert_repos(cursor, r[0], participant, utcnow())
next_page = r[2].get('next')
if not next_page:
break
sleep(1)
deleted_count = cursor.one("""
WITH deleted AS (
DELETE FROM repositories
WHERE participant = %s
AND platform = %s
AND info_fetched_at < %s
RETURNING id
)
SELECT count(*) FROM deleted
""", (participant.id, account.platform, start_time))
event_type = 'fetch_repos:%s' % account.id
payload = dict(partial_list=bool(next_page), deleted_count=deleted_count)
participant.add_event(cursor, event_type, payload)
cursor.run("""
DELETE FROM events
WHERE participant = %s
AND type = %s
AND (NOT payload ? 'deleted_count' OR payload->'deleted_count' = '0')
AND ts < (current_timestamp - interval '6 days')
""", (participant.id, event_type))
except (InvalidGrantError, TokenExpiredError) as e:
logger.debug("The refetch failed: %s" % e)
return
# The update was successful, clean up the rate_limiting table
website.db.run("DELETE FROM rate_limiting WHERE key = %s", (rl_prefix + ':' + rl_key,))