def _refetch_repos_for_account(rl_prefix, rl_key, participant, account): sess = account.get_auth_session() logger.debug( "Refetching profile data for participant ~%s from %s account %s" % (participant.id, account.platform, account.user_id)) try: account = account.refresh_user_info() except (UnableToRefreshAccount, UserNotFound) as e: logger.debug("The refetch failed: %s" % e) sleep(1) logger.debug( "Refetching repository data for participant ~%s from %s account %s" % (participant.id, account.platform, account.user_id)) start_time = utcnow() try: with website.db.get_cursor() as cursor: next_page = None for i in range(10): r = account.platform_data.get_repos(account, page_url=next_page, sess=sess) upsert_repos(cursor, r[0], participant, utcnow()) next_page = r[2].get('next') if not next_page: break sleep(1) deleted_count = cursor.one( """ WITH deleted AS ( DELETE FROM repositories WHERE participant = %s AND platform = %s AND info_fetched_at < %s RETURNING id ) SELECT count(*) FROM deleted """, (participant.id, account.platform, start_time)) event_type = 'fetch_repos:%s' % account.id payload = dict(partial_list=bool(next_page), deleted_count=deleted_count) participant.add_event(cursor, event_type, payload) cursor.run( """ DELETE FROM events WHERE participant = %s AND type = %s AND (NOT payload ? 'deleted_count' OR payload->'deleted_count' = '0') AND ts < (current_timestamp - interval '6 days') """, (participant.id, event_type)) except (InvalidGrantError, TokenExpiredError) as e: logger.debug("The refetch failed: %s" % e) return # The update was successful, clean up the rate_limiting table website.db.run("DELETE FROM rate_limiting WHERE key = %s", (rl_prefix + ':' + rl_key, ))
def test_dispute_callback_lost(self, save, get_payin, get_dispute): self.make_participant( 'LiberapayOrg', kind='organization', balance=EUR('100.00'), mangopay_user_id='0', mangopay_wallet_id='0', ) save.side_effect = fake_transfer e_id = self.make_exchange('mango-cc', EUR('16'), EUR('1'), self.janet) dispute = Dispute() dispute.Id = '-1' dispute.CreationDate = utcnow() dispute.DisputedFunds = Money(1700, 'EUR') dispute.DisputeType = 'CONTESTABLE' dispute.InitialTransactionType = 'PAYIN' get_dispute.return_value = dispute payin = PayIn(tag=str(e_id)) get_payin.return_value = payin # Transfer some of the money to homer self.janet.set_tip_to(self.homer, EUR('3.68')) Payday.start().run() # Withdraw some of the money self.make_exchange('mango-ba', EUR('-2.68'), 0, self.homer) # Add a bit of money that will remain undisputed, to test bundle swapping self.make_exchange('mango-cc', EUR('0.32'), 0, self.janet) self.make_exchange('mango-cc', EUR('0.55'), 0, self.homer) # Call back self.db.self_check() for status in ('CREATED', 'CLOSED'): dispute.Status = status if status == 'CLOSED': dispute.ResultCode = 'LOST' qs = "EventType=DISPUTE_" + status + "&RessourceId=123456790" r = self.callback(qs, raise_immediately=True) assert r.code == 200, r.text self.db.self_check() # Check final state balances = dict( self.db.all("SELECT username, balance FROM participants")) assert balances == { '_chargebacks_': EUR('16.00'), 'david': 0, 'homer': 0, 'janet': 0, 'LiberapayOrg': EUR('98.19'), } debts = dict(((r[0], r[1]), r[2]) for r in self.db.all(""" SELECT p_debtor.username AS debtor, p_creditor.username AS creditor, sum(d.amount) FROM debts d JOIN participants p_debtor ON p_debtor.id = d.debtor JOIN participants p_creditor ON p_creditor.id = d.creditor WHERE d.status = 'due' GROUP BY p_debtor.username, p_creditor.username """)) assert debts == { ('janet', 'LiberapayOrg'): EUR('1.00'), ('janet', 'homer'): EUR('3.36'), ('homer', 'LiberapayOrg'): EUR('1.81'), }
def refetch_repos(): with website.db.get_cursor() as cursor: repo = cursor.one(""" SELECT r.participant, r.platform FROM repositories r WHERE r.info_fetched_at < now() - interval '6 days' AND r.participant IS NOT NULL ORDER BY r.info_fetched_at ASC LIMIT 1 """) if not repo: return participant = Participant.from_id(repo.participant) account = participant.get_account_elsewhere(repo.platform) sess = account.get_auth_session() start_time = utcnow() logger.debug( "Refetching repository data for participant ~%s from %s account %s" % (participant.id, account.platform, account.user_id)) next_page = None for i in range(10): r = account.platform_data.get_repos(account, page_url=next_page, sess=sess) upsert_repos(cursor, r[0], participant, utcnow()) next_page = r[2].get('next') if not next_page: break sleep(1) deleted_count = cursor.one( """ WITH deleted AS ( DELETE FROM repositories WHERE participant = %s AND platform = %s AND info_fetched_at < %s RETURNING id ) SELECT count(*) FROM deleted """, (participant.id, account.platform, start_time)) event_type = 'fetch_repos:%s' % account.id payload = dict(partial_list=bool(next_page), deleted_count=deleted_count) participant.add_event(cursor, event_type, payload)
def test_dispute_callback_lost(self, save, get_payin, get_dispute): self.make_participant( 'LiberapayOrg', kind='organization', balance=D('100.00'), mangopay_user_id='0', mangopay_wallet_id='0', ) save.side_effect = fake_transfer e_id = self.make_exchange('mango-cc', D('16'), D('1'), self.janet) dispute = Dispute() dispute.Id = '-1' dispute.CreationDate = utcnow() dispute.DisputedFunds = Money(1700, 'EUR') dispute.DisputeType = 'CONTESTABLE' dispute.InitialTransactionType = 'PAYIN' get_dispute.return_value = dispute payin = PayIn(tag=str(e_id)) get_payin.return_value = payin # Transfer some of the money to homer self.janet.set_tip_to(self.homer, EUR('3.68')) Payday.start().run() # Withdraw some of the money self.make_exchange('mango-ba', D('-2.68'), 0, self.homer) # Add a bit of money that will remain undisputed, to test bundle swapping self.make_exchange('mango-cc', D('0.32'), 0, self.janet) self.make_exchange('mango-cc', D('0.55'), 0, self.homer) # Call back self.db.self_check() for status in ('CREATED', 'CLOSED'): dispute.Status = status if status == 'CLOSED': dispute.ResultCode = 'LOST' qs = "EventType=DISPUTE_"+status+"&RessourceId=123456790" r = self.callback(qs, raise_immediately=True) assert r.code == 200, r.text self.db.self_check() # Check final state balances = dict(self.db.all("SELECT username, balance FROM participants")) assert balances == { '_chargebacks_': D('16.00'), 'david': 0, 'homer': 0, 'janet': 0, 'LiberapayOrg': D('98.19'), } debts = dict(((r[0], r[1]), r[2]) for r in self.db.all(""" SELECT p_debtor.username AS debtor, p_creditor.username AS creditor, sum(d.amount) FROM debts d JOIN participants p_debtor ON p_debtor.id = d.debtor JOIN participants p_creditor ON p_creditor.id = d.creditor WHERE d.status = 'due' GROUP BY p_debtor.username, p_creditor.username """)) assert debts == { ('janet', 'LiberapayOrg'): D('1.00'), ('janet', 'homer'): D('3.36'), ('homer', 'LiberapayOrg'): D('1.81'), }
def refetch_repos(): with website.db.get_cursor() as cursor: repo = cursor.one(""" SELECT r.participant, r.platform FROM repositories r WHERE r.info_fetched_at < now() - interval '6 days' AND r.participant IS NOT NULL ORDER BY r.info_fetched_at ASC LIMIT 1 """) if not repo: return participant = Participant.from_id(repo.participant) account = participant.get_account_elsewhere(repo.platform) sess = account.get_auth_session() start_time = utcnow() logger.debug( "Refetching repository data for participant ~%s from %s account %s" % (participant.id, account.platform, account.user_id) ) next_page = None for i in range(10): r = account.platform_data.get_repos(account, page_url=next_page, sess=sess) upsert_repos(cursor, r[0], participant, utcnow()) next_page = r[2].get('next') if not next_page: break sleep(1) deleted_count = cursor.one(""" WITH deleted AS ( DELETE FROM repositories WHERE participant = %s AND platform = %s AND info_fetched_at < %s RETURNING id ) SELECT count(*) FROM deleted """, (participant.id, account.platform, start_time)) event_type = 'fetch_repos:%s' % account.id payload = dict(partial_list=bool(next_page), deleted_count=deleted_count) participant.add_event(cursor, event_type, payload)
def test_dispute_callback_won(self, save, get_payin, get_dispute): self.make_participant('LiberapayOrg', kind='organization') save.side_effect = fake_transfer e_id = self.make_exchange('mango-cc', EUR('16'), EUR('1'), self.janet) dispute = Dispute() dispute.Id = '-1' dispute.CreationDate = utcnow() dispute.DisputedFunds = Money(1700, 'EUR') dispute.DisputeType = 'CONTESTABLE' dispute.InitialTransactionType = 'PAYIN' get_dispute.return_value = dispute payin = PayIn(tag=str(e_id)) get_payin.return_value = payin # Transfer some of the money to homer self.janet.set_tip_to(self.homer, EUR('3.68')) Payday.start().run() # Withdraw some of the money self.make_exchange('mango-ba', EUR('-2.68'), 0, self.homer) # Add money that will remain undisputed, to test bundle swapping self.make_exchange('mango-cc', EUR('2.69'), 0, self.janet) # Call back self.db.self_check() for status in ('CREATED', 'CLOSED'): dispute.Status = status if status == 'CLOSED': dispute.ResultCode = 'WON' qs = "EventType=DISPUTE_" + status + "&RessourceId=123456790" r = self.callback(qs) assert r.code == 200, r.text self.db.self_check() # Check final state disputed = self.db.all("SELECT * FROM cash_bundles WHERE disputed") debts = self.db.all("SELECT * FROM debts") assert not disputed assert not debts balances = dict( self.db.all("SELECT username, balance FROM participants")) assert balances == { 'david': 0, 'homer': EUR('1.00'), 'janet': EUR('15.01'), 'LiberapayOrg': 0, }
def test_dispute_callback_won(self, save, get_payin, get_dispute): self.make_participant('LiberapayOrg', kind='organization') save.side_effect = fake_transfer e_id = self.make_exchange('mango-cc', D('16'), D('1'), self.janet) dispute = Dispute() dispute.Id = '-1' dispute.CreationDate = utcnow() dispute.DisputedFunds = Money(1700, 'EUR') dispute.DisputeType = 'CONTESTABLE' dispute.InitialTransactionType = 'PAYIN' get_dispute.return_value = dispute payin = PayIn(tag=str(e_id)) get_payin.return_value = payin # Transfer some of the money to homer self.janet.set_tip_to(self.homer, EUR('3.68')) Payday.start().run() # Withdraw some of the money self.make_exchange('mango-ba', D('-2.68'), 0, self.homer) # Add money that will remain undisputed, to test bundle swapping self.make_exchange('mango-cc', D('2.69'), 0, self.janet) # Call back self.db.self_check() for status in ('CREATED', 'CLOSED'): dispute.Status = status if status == 'CLOSED': dispute.ResultCode = 'WON' qs = "EventType=DISPUTE_"+status+"&RessourceId=123456790" r = self.callback(qs) assert r.code == 200, r.text self.db.self_check() # Check final state disputed = self.db.all("SELECT * FROM cash_bundles WHERE disputed") debts = self.db.all("SELECT * FROM debts") assert not disputed assert not debts balances = dict(self.db.all("SELECT username, balance FROM participants")) assert balances == { 'david': 0, 'homer': D('1.00'), 'janet': D('15.01'), 'LiberapayOrg': 0, }
def refetch_repos(): # Note: the rate_limiting table is used to avoid blocking on errors rl_prefix = 'refetch_repos' rl_cap, rl_period = RATE_LIMITS[rl_prefix] repo = website.db.one( """ SELECT r.participant, r.platform FROM repositories r WHERE r.info_fetched_at < now() - interval '6 days' AND r.participant IS NOT NULL AND r.show_on_profile AND check_rate_limit(%s || r.participant::text || ':' || r.platform, %s, %s) ORDER BY r.info_fetched_at ASC LIMIT 1 """, (rl_prefix + ':', rl_cap, rl_period)) if not repo: return rl_key = '%s:%s' % (repo.participant, repo.platform) website.db.hit_rate_limit(rl_prefix, rl_key) participant = Participant.from_id(repo.participant) account = participant.get_account_elsewhere(repo.platform) sess = account.get_auth_session() logger.debug( "Refetching profile data for participant ~%s from %s account %s" % (participant.id, account.platform, account.user_id)) account = account.refresh_user_info() sleep(1) logger.debug( "Refetching repository data for participant ~%s from %s account %s" % (participant.id, account.platform, account.user_id)) start_time = utcnow() with website.db.get_cursor() as cursor: next_page = None for i in range(10): r = account.platform_data.get_repos(account, page_url=next_page, sess=sess) upsert_repos(cursor, r[0], participant, utcnow()) next_page = r[2].get('next') if not next_page: break sleep(1) deleted_count = cursor.one( """ WITH deleted AS ( DELETE FROM repositories WHERE participant = %s AND platform = %s AND info_fetched_at < %s RETURNING id ) SELECT count(*) FROM deleted """, (participant.id, account.platform, start_time)) event_type = 'fetch_repos:%s' % account.id payload = dict(partial_list=bool(next_page), deleted_count=deleted_count) participant.add_event(cursor, event_type, payload) cursor.run( """ DELETE FROM events WHERE participant = %s AND type = %s AND (NOT payload ? 'deleted_count' OR payload->'deleted_count' = '0') AND ts < (current_timestamp - interval '6 days') """, (participant.id, event_type)) # The update was successful, clean up the rate_limiting table website.db.run("DELETE FROM rate_limiting WHERE key = %s", (rl_prefix + ':' + rl_key, ))
def refetch_repos(): # Note: the rate_limiting table is used to avoid blocking on errors rl_prefix = 'refetch_repos' rl_cap, rl_period = RATE_LIMITS[rl_prefix] repo = website.db.one(""" SELECT r.participant, r.platform FROM repositories r WHERE r.info_fetched_at < now() - interval '6 days' AND r.participant IS NOT NULL AND r.show_on_profile AND check_rate_limit(%s || r.participant::text || ':' || r.platform, %s, %s) ORDER BY r.info_fetched_at ASC LIMIT 1 """, (rl_prefix + ':', rl_cap, rl_period)) if not repo: return rl_key = '%s:%s' % (repo.participant, repo.platform) website.db.hit_rate_limit(rl_prefix, rl_key) participant = Participant.from_id(repo.participant) account = participant.get_account_elsewhere(repo.platform) if not account: return sess = account.get_auth_session() logger.debug( "Refetching profile data for participant ~%s from %s account %s" % (participant.id, account.platform, account.user_id) ) try: account = account.refresh_user_info() except (UnableToRefreshAccount, UserNotFound) as e: logger.debug("The refetch failed: %s" % e) sleep(1) logger.debug( "Refetching repository data for participant ~%s from %s account %s" % (participant.id, account.platform, account.user_id) ) start_time = utcnow() try: with website.db.get_cursor() as cursor: next_page = None for i in range(10): r = account.platform_data.get_repos(account, page_url=next_page, sess=sess) upsert_repos(cursor, r[0], participant, utcnow()) next_page = r[2].get('next') if not next_page: break sleep(1) deleted_count = cursor.one(""" WITH deleted AS ( DELETE FROM repositories WHERE participant = %s AND platform = %s AND info_fetched_at < %s RETURNING id ) SELECT count(*) FROM deleted """, (participant.id, account.platform, start_time)) event_type = 'fetch_repos:%s' % account.id payload = dict(partial_list=bool(next_page), deleted_count=deleted_count) participant.add_event(cursor, event_type, payload) cursor.run(""" DELETE FROM events WHERE participant = %s AND type = %s AND (NOT payload ? 'deleted_count' OR payload->'deleted_count' = '0') AND ts < (current_timestamp - interval '6 days') """, (participant.id, event_type)) except (InvalidGrantError, TokenExpiredError) as e: logger.debug("The refetch failed: %s" % e) return # The update was successful, clean up the rate_limiting table website.db.run("DELETE FROM rate_limiting WHERE key = %s", (rl_prefix + ':' + rl_key,))