def log_result_of_request(website, response, request):
"""Log access. With our own format (not Apache's).
"""
if website.logging_threshold > 0: # short-circuit
return
# What was the URL path translated to?
# ====================================
fs = getattr(request, 'fs', '')
if fs.startswith(website.www_root):
fs = fs[len(website.www_root):]
if fs:
fs = '.'+fs
else:
fs = '...' + fs[-21:]
msg = "%-24s %s" % (request.line.uri.path.raw, fs)
# Where was response raised from?
# ===============================
filename, linenum = response.whence_raised()
if filename is not None:
response = "%s (%s:%d)" % (response, filename, linenum)
else:
response = str(response)
# Log it.
# =======
aspen.log("%-36s %s" % (response, msg))
def update_stats(self):
self.db.run("""\
WITH our_transfers AS (
SELECT *
FROM transfers
WHERE "timestamp" >= %(ts_start)s
)
, our_tips AS (
SELECT *
FROM our_transfers
WHERE context = 'tip'
)
, our_pachinkos AS (
SELECT *
FROM our_transfers
WHERE context = 'take'
)
, our_exchanges AS (
SELECT *
FROM exchanges
WHERE "timestamp" >= %(ts_start)s
)
, our_achs AS (
SELECT *
FROM our_exchanges
WHERE amount < 0
)
, our_charges AS (
SELECT *
FROM our_exchanges
WHERE amount > 0
)
UPDATE paydays
SET nactive = (
SELECT DISTINCT count(*) FROM (
SELECT tipper FROM our_transfers
UNION
SELECT tippee FROM our_transfers
) AS foo
)
, ntippers = (SELECT count(DISTINCT tipper) FROM our_transfers)
, ntips = (SELECT count(*) FROM our_tips)
, npachinko = (SELECT count(*) FROM our_pachinkos)
, pachinko_volume = (SELECT COALESCE(sum(amount), 0) FROM our_pachinkos)
, ntransfers = (SELECT count(*) FROM our_transfers)
, transfer_volume = (SELECT COALESCE(sum(amount), 0) FROM our_transfers)
, nachs = (SELECT count(*) FROM our_achs)
, ach_volume = (SELECT COALESCE(sum(amount), 0) FROM our_achs)
, ach_fees_volume = (SELECT COALESCE(sum(fee), 0) FROM our_achs)
, ncharges = (SELECT count(*) FROM our_charges)
, charge_volume = (
SELECT COALESCE(sum(amount + fee), 0)
FROM our_charges
)
, charge_fees_volume = (SELECT COALESCE(sum(fee), 0) FROM our_charges)
WHERE ts_end='1970-01-01T00:00:00+00'::timestamptz
""", {'ts_start': self.ts_start})
log("Updated payday stats.")
def capture_card_hold(db, participant, amount, hold):
"""Capture the previously created hold on the participant's credit card.
"""
typecheck( hold, braintree.Transaction
, amount, Decimal
)
username = participant.username
assert participant.id == int(hold.custom_fields['participant_id'])
route = ExchangeRoute.from_address(participant, 'braintree-cc', hold.credit_card['token'])
assert isinstance(route, ExchangeRoute)
cents, amount_str, charge_amount, fee = _prep_hit(amount)
amount = charge_amount - fee # account for possible rounding
e_id = record_exchange(db, route, amount, fee, participant, 'pre')
# TODO: Find a way to link transactions and corresponding exchanges
# meta = dict(participant_id=participant.id, exchange_id=e_id)
error = ''
try:
result = braintree.Transaction.submit_for_settlement(hold.id, str(cents/100.00))
assert result.is_success
if result.transaction.status != 'submitted_for_settlement':
error = result.transaction.status
except Exception as e:
error = repr_exception(e)
if error == '':
record_exchange_result(db, e_id, 'succeeded', None, participant)
log("Captured " + amount_str + " on Braintree for " + username)
else:
record_exchange_result(db, e_id, 'failed', error, participant)
raise Exception(error)
def payday():
# Wire things up.
# ===============
env = wireup.env()
wireup.db(env)
wireup.billing(env)
# Lazily import the billing module.
# =================================
# This dodges a problem where db in billing is None if we import it from
# gratipay before calling wireup.billing.
from gratipay.billing.payday import Payday
try:
Payday.start().run()
except KeyboardInterrupt:
pass
except:
import aspen
import traceback
aspen.log(traceback.format_exc())
def finish_payday():
# Finish Payday.
# ==============
# Transfer pending into balance for all users, setting pending to NULL.
# Close out the paydays entry as well.
with db.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""\
UPDATE participants
SET balance = (balance + pending)
, pending = NULL
""")
cursor.execute("""\
UPDATE paydays
SET ts_end=now()
WHERE ts_end='1970-01-01T00:00:00+00'::timestamptz
RETURNING id
""")
assert_one_payday(cursor.fetchone())
conn.commit()
log("Finished payday.")
def payday():
# Wire things up.
# ===============
# Manually override max db connections so that we only have one connection.
# Our db access is serialized right now anyway, and with only one
# connection it's easier to trust changes to statement_timeout. The point
# here is that we want to turn off statement_timeout for payday.
env = wireup.env()
env.database_maxconn = 1
db = wireup.db(env)
db.run("SET statement_timeout = 0")
wireup.billing(env)
wireup.nanswers(env)
# Lazily import the billing module.
# =================================
# This dodges a problem where db in billing is None if we import it from
# gittip before calling wireup.billing.
from gittip.billing.payday import Payday
try:
Payday(db).run()
except KeyboardInterrupt:
pass
except:
import aspen
import traceback
aspen.log(traceback.format_exc())
def capture_card_hold(db, participant, amount, hold):
"""Capture the previously created hold on the participant's credit card.
"""
typecheck( hold, balanced.CardHold
, amount, Decimal
)
username = participant.username
assert participant.id == int(hold.meta['participant_id'])
route = ExchangeRoute.from_address(participant, 'balanced-cc', hold.card_href)
assert isinstance(route, ExchangeRoute)
cents, amount_str, charge_amount, fee = _prep_hit(amount)
amount = charge_amount - fee # account for possible rounding
e_id = record_exchange(db, route, amount, fee, participant, 'pre')
meta = dict(participant_id=participant.id, exchange_id=e_id)
try:
hold.capture(amount=cents, description=username, meta=meta)
record_exchange_result(db, e_id, 'succeeded', None, participant)
except Exception as e:
error = repr_exception(e)
record_exchange_result(db, e_id, 'failed', error, participant)
raise
hold.meta['state'] = 'captured'
hold.save()
log("Captured " + amount_str + " on Balanced for " + username)
def payday():
"""This is the big one.
Settling the graph of Gittip balances is an abstract event called Payday.
On Payday, we want to use a participant's Gittip balance to settle their
tips due (pulling in more money via credit card as needed), but we only
want to use their balance at the start of Payday. Balance changes should be
atomic globally per-Payday.
This function runs every Friday. It is structured such that it can be run
again safely if it crashes.
"""
log("Greetings, program! It's PAYDAY!!!!")
participants, payday_start = initialize_payday()
# Drop to core.
# =============
# We are now locked for Payday. If the power goes out at this point then we
# will need to start over and reacquire the lock.
payday_loop(payday_start, participants)
finish_payday()
def payday():
# Wire things up.
# ===============
env = wireup.env()
db = wireup.db(env)
wireup.billing(env)
# Lazily import the billing module.
# =================================
from liberapay.billing.exchanges import sync_with_mangopay
from liberapay.billing.payday import Payday
try:
sync_with_mangopay(db)
Payday.start().run()
except KeyboardInterrupt:
pass
except:
import aspen
import traceback
aspen.log(traceback.format_exc())
def update_balances(cursor):
participants = cursor.all("""
UPDATE participants p
SET balance = (balance + p2.new_balance - p2.old_balance)
FROM payday_participants p2
WHERE p.id = p2.id
AND p2.new_balance <> p2.old_balance
RETURNING p.id
, p.username
, balance AS new_balance
, ( SELECT balance
FROM participants p3
WHERE p3.id = p.id
) AS cur_balance;
""")
# Check that balances aren't becoming (more) negative
for p in participants:
if p.new_balance < 0 and p.new_balance < p.cur_balance:
log(p)
raise NegativeBalance()
cursor.run("""
INSERT INTO transfers (timestamp, tipper, tippee, amount, context)
SELECT * FROM payday_transfers;
""")
log("Updated the balances of %i participants." % len(participants))
def check_ratelimit_headers(self, response):
"""Emit log messages if we're running out of ratelimit.
"""
prefix = getattr(self, 'ratelimit_headers_prefix', None)
if prefix:
limit = response.headers.get(prefix+'limit')
remaining = response.headers.get(prefix+'remaining')
reset = response.headers.get(prefix+'reset')
try:
limit, remaining, reset = int(limit), int(remaining), int(reset)
except (TypeError, ValueError):
limit, remaining, reset = None, None, None
if None in (limit, remaining, reset):
d = dict(limit=limit, remaining=remaining, reset=reset)
log('Got weird rate headers from %s: %s' % (self.name, d))
else:
percent_remaining = remaining/limit
if percent_remaining < 0.5:
reset = to_age(datetime.fromtimestamp(reset, tz=utc))
log_msg = (
'{0} API: {1:.1%} of ratelimit has been consumed, '
'{2} requests remaining, resets {3}.'
).format(self.name, 1 - percent_remaining, remaining, reset)
log_lvl = logging.WARNING
if percent_remaining < 0.2:
log_lvl = logging.ERROR
elif percent_remaining < 0.05:
log_lvl = logging.CRITICAL
log(log_msg, log_lvl)
def run(self):
"""This is the starting point for payday.
This method runs every Thursday. It is structured such that it can be
run again safely (with a newly-instantiated Payday object) if it
crashes.
"""
self.db.self_check()
_start = aspen.utils.utcnow()
log("Greetings, program! It's PAYDAY!!!!")
if self.stage < 1:
self.payin()
self.mark_stage_done()
if self.stage < 2:
self.payout()
self.mark_stage_done()
if self.stage < 3:
self.update_stats()
self.update_cached_amounts()
self.mark_stage_done()
self.end()
_end = aspen.utils.utcnow()
_delta = _end - _start
fmt_past = "Script ran for %%(age)s (%s)." % _delta
log(aspen.utils.to_age(_start, fmt_past=fmt_past))
def get_user_info(login):
"""Get the given user's information from the DB or failing that, github.
:param login:
A unicode string representing a username in github.
:returns:
A dictionary containing github specific information for the user.
"""
typecheck(login, unicode)
rec = gittip.db.fetchone( "SELECT user_info FROM elsewhere "
"WHERE platform='github' "
"AND user_info->'login' = %s"
, (login,)
)
if rec is not None:
user_info = rec['user_info']
else:
url = "https://api.github.com/users/%s"
user_info = requests.get(url % login)
status = user_info.status_code
content = user_info.text
if status == 200:
user_info = json.loads(content)
elif status == 404:
raise Response(404,
"GitHub identity '{0}' not found.".format(login))
else:
log("Github api responded with {0}: {1}".format(status, content),
level=logging.WARNING)
raise Response(502, "GitHub lookup failed with %d." % status)
return user_info
def charge_on_balanced(self, username, balanced_customer_href, amount):
"""We have a purported balanced_customer_href. Try to use it.
"""
typecheck( username, unicode
, balanced_customer_href, unicode
, amount, Decimal
)
cents, msg, charge_amount, fee = self._prep_hit(amount)
msg = msg % (username, "Balanced")
try:
customer = balanced.Customer.fetch(balanced_customer_href)
customer.cards.one().debit(amount=cents, description=username)
log(msg + "succeeded.")
error = ""
except balanced.exc.HTTPError as err:
error = err.message.message
except:
error = repr(sys.exc_info()[1])
if error:
log(msg + "failed: %s" % error)
return charge_amount, fee, error
def get_user_info(db, username):
"""Get the given user's information from the DB or failing that, bitbucket.
:param username:
A unicode string representing a username in bitbucket.
:returns:
A dictionary containing bitbucket specific information for the user.
"""
typecheck(username, (unicode, PathPart))
rec = db.one(
"""
SELECT user_info FROM elsewhere
WHERE platform='bitbucket'
AND user_info->'username' = %s
""",
(username,),
)
if rec is not None:
user_info = rec
else:
url = "%s/users/%s?pagelen=100"
user_info = requests.get(url % (BASE_API_URL, username))
status = user_info.status_code
content = user_info.content
if status == 200:
user_info = json.loads(content)["user"]
elif status == 404:
raise Response(404, "Bitbucket identity '{0}' not found.".format(username))
else:
log("Bitbucket api responded with {0}: {1}".format(status, content), level=logging.WARNING)
raise Response(502, "Bitbucket lookup failed with %d." % status)
return user_info
def run(self):
"""This is the starting point for payday.
This method runs every Thursday. It is structured such that it can be
run again safely (with a newly-instantiated Payday object) if it
crashes.
"""
self.db.self_check()
_start = aspen.utils.utcnow()
log("Greetings, program! It's PAYDAY!!!!")
ts_start = self.start()
self.zero_out_pending(ts_start)
self.payin(ts_start, self.genparticipants(ts_start, loop=LOOP_PAYIN))
self.move_pending_to_balance_for_teams()
self.pachinko(ts_start, self.genparticipants(ts_start, loop=LOOP_PACHINKO))
self.clear_pending_to_balance()
self.payout(ts_start, self.genparticipants(ts_start, loop=LOOP_PAYOUT))
self.update_stats(ts_start)
self.update_receiving_amounts()
self.end()
self.db.self_check()
_end = aspen.utils.utcnow()
_delta = _end - _start
fmt_past = "Script ran for {age} (%s)." % _delta
log(aspen.utils.to_age(_start, fmt_past=fmt_past))
def hit_balanced(self, participant_id, balanced_account_uri, amount):
"""We have a purported balanced_account_uri. Try to use it.
"""
typecheck( participant_id, unicode
, balanced_account_uri, unicode
, amount, Decimal
)
try_charge_amount = (amount + FEE[0]) * FEE[1]
try_charge_amount = try_charge_amount.quantize( FEE[0]
, rounding=ROUND_UP
)
charge_amount = try_charge_amount
also_log = ''
if charge_amount < MINIMUM:
charge_amount = MINIMUM # per Balanced
also_log = ', rounded up to $%s' % charge_amount
fee = try_charge_amount - amount
cents = int(charge_amount * 100)
msg = "Charging %s %d cents ($%s + $%s fee = $%s%s) ... "
msg %= participant_id, cents, amount, fee, try_charge_amount, also_log
try:
customer = balanced.Account.find(balanced_account_uri)
customer.debit(cents, description=participant_id)
log(msg + "succeeded.")
except balanced.exc.HTTPError as err:
log(msg + "failed: %s" % err.message)
return charge_amount, fee, err.message
return charge_amount, fee, None
def run(self):
"""This is the starting point for payday.
This method runs every Thursday. It is structured such that it can be
run again safely (with a newly-instantiated Payday object) if it
crashes.
"""
log("Greetings, program! It's PAYDAY!!!!")
ts_start = self.start()
self.zero_out_pending()
def genparticipants(ts_start):
"""Closure generator to yield participants with tips and total.
We re-fetch participants each time, because the second time through
we want to use the total obligations they have for next week, and
if we pass a non-False ts_start to get_tips_and_total then we only
get unfulfilled tips from prior to that timestamp, which is none of
them by definition.
"""
for participant in self.get_participants():
tips, total = get_tips_and_total( participant['id']
, for_payday=ts_start
, db=self.db
)
typecheck(total, Decimal)
yield(participant, tips, total)
self.payin(ts_start, genparticipants(ts_start))
self.clear_pending_to_balance()
self.payout(ts_start, genparticipants(False))
self.end()
def payday():
# Wire things up.
# ===============
env = wireup.env()
db = wireup.db(env)
wireup.billing(env)
wireup.nanswers(env)
# Lazily import the billing module.
# =================================
# This dodges a problem where db in billing is None if we import it from
# gittip before calling wireup.billing.
from gittip.billing.exchanges import sync_with_balanced
from gittip.billing.payday import Payday
try:
with db.get_cursor() as cursor:
sync_with_balanced(cursor)
Payday.start().run()
except KeyboardInterrupt:
pass
except:
import aspen
import traceback
aspen.log(traceback.format_exc())
def run(self, log_dir='', keep_log=False):
"""This is the starting point for payday.
It is structured such that it can be run again safely (with a
newly-instantiated Payday object) if it crashes.
"""
self.db.self_check()
_start = aspen.utils.utcnow()
log("Greetings, program! It's PAYDAY!!!!")
self.shuffle(log_dir)
self.update_stats()
self.update_cached_amounts()
self.end()
self.notify_participants()
if not keep_log:
os.unlink(self.transfers_filename)
_end = aspen.utils.utcnow()
_delta = _end - _start
fmt_past = "Script ran for %%(age)s (%s)." % _delta
log(aspen.utils.to_age(_start, fmt_past=fmt_past))
def get_user_info(db, username, osm_api_url):
"""Get the given user's information from the DB or failing that, openstreetmap.
:param username:
A unicode string representing a username in OpenStreetMap.
:param osm_api_url:
URL of OpenStreetMap API.
:returns:
A dictionary containing OpenStreetMap specific information for the user.
"""
typecheck(username, (unicode, PathPart))
rec = db.one("""
SELECT user_info FROM elsewhere
WHERE platform='openstreetmap'
AND user_info->'username' = %s
""", (username,))
if rec is not None:
user_info = rec
else:
osm_user = requests.get("%s/user/%s" % (osm_api_url, username))
if osm_user.status_code == 200:
log("User %s found in OpenStreetMap but not in gittip." % username)
user_info = None
elif osm_user.status_code == 404:
raise Response(404,
"OpenStreetMap identity '{0}' not found.".format(username))
else:
log("OpenStreetMap api responded with {0}: {1}".format(status, content),
level=logging.WARNING)
raise Response(502, "OpenStreetMap lookup failed with %d." % status)
return user_info
def process_takes(cursor, ts_start):
log("Processing takes.")
cursor.run("""
UPDATE payday_teams SET available_today = LEAST(available, balance);
INSERT INTO payday_takes
SELECT team_id, participant_id, amount
FROM ( SELECT DISTINCT ON (team_id, participant_id)
team_id, participant_id, amount, ctime
FROM takes
WHERE mtime < %(ts_start)s
ORDER BY team_id, participant_id, mtime DESC
) t
WHERE t.amount > 0
AND t.team_id IN (SELECT id FROM payday_teams)
AND t.participant_id IN (SELECT id FROM payday_participants)
AND ( SELECT ppd.id
FROM payday_payments_done ppd
JOIN participants ON participants.id = t.participant_id
JOIN teams ON teams.id = t.team_id
WHERE participants.username = ppd.participant
AND teams.slug = ppd.team
AND direction = 'to-participant'
) IS NULL
ORDER BY t.team_id, t.amount ASC;
""", dict(ts_start=ts_start))
def create_github_review_issue(self):
"""POST to GitHub, and return the URL of the new issue.
"""
api_url = "https://api.github.com/repos/{}/issues".format(self.review_repo)
data = json.dumps(
{
"title": self.name,
"body": "https://gratipay.com/{}/\n\n".format(self.slug)
+ "(This application will remain open for at least a week.)",
}
)
out = ""
try:
r = requests.post(api_url, auth=self.review_auth, data=data)
if r.status_code == 201:
out = r.json()["html_url"]
else:
log(r.status_code)
log(r.text)
err = str(r.status_code)
except:
err = "eep"
if not out:
out = "https://github.com/gratipay/team-review/issues#error-{}".format(err)
return out
def check_one(filename):
"""Given a filename, return None or restart.
"""
# The file may have been removed from the filesystem.
# ===================================================
if not os.path.isfile(filename):
if filename in mtimes:
aspen.log("file deleted: %s" % filename)
restart()
else:
# We haven't seen the file before. It has probably been loaded
# from a zip (egg) archive.
return
# Or not, in which case, check the modification time.
# ===================================================
mtime = os.stat(filename).st_mtime
if filename not in mtimes: # first time we've seen it
mtimes[filename] = mtime
if mtime > mtimes[filename]:
aspen.log("file changed: %s" % filename)
restart()
def install_restarter_for_website(website):
"""
"""
if website.changes_reload:
aspen.log("Aspen will restart when configuration scripts or "
"Python modules change.")
execution.install(website)
def run(self):
"""This is the starting point for payday.
This method runs every Thursday. It is structured such that it can be
run again safely (with a newly-instantiated Payday object) if it
crashes.
"""
_start = aspen.utils.utcnow()
log("Greetings, program! It's PAYDAY!!!!")
ts_start = self.start()
self.zero_out_pending(ts_start)
self.payin(ts_start, self.genparticipants(ts_start, ts_start))
self.move_pending_to_balance_for_teams()
self.pachinko(ts_start, self.genparticipants(ts_start, ts_start))
self.clear_pending_to_balance()
self.payout(ts_start, self.genparticipants(ts_start, False))
self.end()
_end = aspen.utils.utcnow()
_delta = _end - _start
fmt_past = "Script ran for {age} (%s)." % _delta
# XXX For some reason newer versions of aspen use old string
# formatting, so if/when we upgrade this will break. Why do we do that
# in aspen, anyway?
log(aspen.utils.to_age(_start, fmt_past=fmt_past))
def create_card_hold(db, participant, amount):
"""Create a hold on the participant's credit card.
Amount should be the nominal amount. We'll compute Gratipay's fee below
this function and add it to amount to end up with charge_amount.
"""
typecheck(amount, Decimal)
username = participant.username
# Perform some last-minute checks.
# ================================
if participant.is_suspicious is not False:
raise NotWhitelisted # Participant not trusted.
route = ExchangeRoute.from_network(participant, 'braintree-cc')
if not route:
return None, 'No credit card'
# Go to Braintree.
# ================
cents, amount_str, charge_amount, fee = _prep_hit(amount)
amount = charge_amount - fee
msg = "Holding " + amount_str + " on Braintree for " + username + " ... "
hold = None
error = ""
try:
result = braintree.Transaction.sale({
'amount': str(cents/100.0),
'customer_id': route.participant.braintree_customer_id,
'payment_method_token': route.address,
'options': { 'submit_for_settlement': False },
'custom_fields': {'participant_id': participant.id}
})
if result.is_success and result.transaction.status == 'authorized':
error = ""
hold = result.transaction
elif result.is_success:
error = "Transaction status was %s" % result.transaction.status
else:
error = result.message
except Exception as e:
error = repr_exception(e)
if error == '':
log(msg + "succeeded.")
else:
log(msg + "failed: %s" % error)
record_exchange(db, route, amount, fee, participant, 'failed', error)
return hold, error
def credit(route):
if route.participant.is_suspicious is None:
log("UNREVIEWED: %s" % route.participant.username)
return
withhold = route.participant.giving
error = ach_credit(self.db, route.participant, withhold)
if error:
self.mark_ach_failed()
def credit(participant):
if participant.is_suspicious is None:
log("UNREVIEWED: %s" % participant.username)
return
withhold = participant.giving + participant.pledging
error = ach_credit(self.db, participant, withhold)
if error:
self.mark_ach_failed()
def cancel_card_hold(hold):
"""Cancel the previously created hold on the participant's credit card.
"""
result = braintree.Transaction.void(hold.id)
assert result.is_success
amount = hold.amount
participant_id = hold.custom_fields['participant_id']
log("Canceled a ${:.2f} hold for {}.".format(amount, participant_id))
def move_pending_to_balance_for_teams(self):
"""Transfer pending into balance for teams.
We do this because debit_participant operates against balance, not
pending. This is because credit card charges go directly into balance
on the first (payin) loop.
"""
self.db.run("""\
UPDATE participants
SET balance = (balance + pending)
, pending = 0
WHERE pending IS NOT NULL
AND number='plural'
""")
# "Moved" instead of "cleared" because we don't also set to null.
log("Moved pending to balance for teams. Ready for pachinko.")
def settle_card_holds(self, cursor, holds):
participants = cursor.all("""
SELECT *
FROM payday_participants
WHERE new_balance < 0
""")
participants = [p for p in participants if p.id in holds]
# Capture holds to bring balances back up to (at least) zero
def capture(p):
amount = -p.new_balance
capture_card_hold(self.db, p, amount, holds.pop(p.id))
threaded_map(capture, participants)
log("Captured %i card holds." % len(participants))
# Cancel the remaining holds
threaded_map(cancel_card_hold, holds.values())
log("Canceled %i card holds." % len(holds))
def __call__(self, period, func, exclusive=False):
if period <= 0:
log('Cron: not installing {}.'.format(func.__name__))
return
log('Cron: installing {} to run every {} seconds{}.'.format(
func.__name__, period, ' with a lock' if exclusive else ''))
if exclusive and not self.has_lock:
self.exclusive_jobs.append((period, func))
self._wait_for_lock()
return
def f():
while True:
try:
func()
except Exception, e:
self.website.tell_sentry(e, {})
log_dammit(traceback.format_exc().strip())
sleep(period)
def create_card_hold(db, participant, amount):
"""Create a hold on the participant's credit card.
Amount should be the nominal amount. We'll compute Gratipay's fee below
this function and add it to amount to end up with charge_amount.
"""
typecheck(amount, Decimal)
username = participant.username
balanced_customer_href = participant.balanced_customer_href
typecheck(username, unicode, balanced_customer_href, (unicode, None))
# Perform some last-minute checks.
# ================================
if balanced_customer_href is None:
raise NoBalancedCustomerHref # Participant has no funding source.
if participant.is_suspicious is not False:
raise NotWhitelisted # Participant not trusted.
# Go to Balanced.
# ===============
cents, amount_str, charge_amount, fee = _prep_hit(amount)
msg = "Holding " + amount_str + " on Balanced for " + username + " ... "
hold = None
try:
card = customer_from_href(balanced_customer_href).cards.one()
hold = card.hold(amount=cents,
description=username,
meta=dict(participant_id=participant.id, state='new'))
log(msg + "succeeded.")
error = ""
except Exception as e:
error = repr_exception(e)
log(msg + "failed: %s" % error)
record_exchange(db, 'bill', amount, fee, participant, 'failed', error)
return hold, error
def f(self, info, *default):
for key in keys:
chain = isinstance(key, basestring) and (key,) or key
try:
v = _getitemchain(info, *chain)
except (KeyError, TypeError):
continue
if v:
v = clean(v)
if not v:
continue
_popitemchain(info, *chain)
return v
if default:
return default[0]
msg = 'Unable to find any of the keys %s in %s API response:\n%s'
msg %= keys, self.name, json.dumps(info, indent=4)
log(msg)
raise KeyError(msg)
def payday():
db = wireup.db()
wireup.billing()
# Lazily import the billing module.
# =================================
# This dodges a problem where db in billing is None if we import it from
# gittip before calling wireup.billing.
from gittip.billing.payday import Payday
try:
Payday(db).run()
except KeyboardInterrupt:
pass
except:
import aspen
import traceback
aspen.log(traceback.format_exc())
def zero_out_pending(self, ts_start):
"""Given a timestamp, zero out the pending column.
We keep track of balance changes as a result of Payday in the pending
column, and then move them over to the balance column in one big
transaction at the end of Payday.
"""
START_PENDING = """\
UPDATE participants
SET pending=0.00
WHERE pending IS NULL
AND claimed_time < %s
"""
self.db.run(START_PENDING, (ts_start, ))
log("Zeroed out the pending column.")
return None
def charge_on_stripe(self, username, stripe_customer_id, amount):
"""We have a purported stripe_customer_id. Try to use it.
"""
typecheck(username, unicode, stripe_customer_id, unicode, amount,
Decimal)
cents, msg, charge_amount, fee = self._prep_hit(amount)
msg = msg % (username, "Stripe")
try:
stripe.Charge.create(customer=stripe_customer_id,
amount=cents,
description=username,
currency="USD")
log(msg + "succeeded.")
error = ""
except stripe.StripeError, err:
error = err.message
log(msg + "failed: %s" % error)
def get_participants(self, ts_start):
"""Given a timestamp, return a list of participants dicts.
"""
PARTICIPANTS = """\
SELECT username
, balance
, balanced_account_uri
, stripe_customer_id
, is_suspicious
, number
FROM participants
WHERE claimed_time IS NOT NULL
AND claimed_time < %s
AND is_suspicious IS NOT true
ORDER BY claimed_time ASC
"""
participants = self.db.all(PARTICIPANTS, (ts_start, ))
log("Fetched participants.")
return participants
def charge_on_balanced(self, username, balanced_account_uri, amount):
"""We have a purported balanced_account_uri. Try to use it.
"""
typecheck(username, unicode, balanced_account_uri, unicode, amount,
Decimal)
cents, msg, charge_amount, fee = self._prep_hit(amount)
msg = msg % (username, "Balanced")
try:
customer = balanced.Account.find(balanced_account_uri)
customer.debit(cents, description=username)
log(msg + "succeeded.")
error = ""
except balanced.exc.HTTPError as err:
error = err.message
log(msg + "failed: %s" % error)
return charge_amount, fee, error
def run(self):
"""This is the starting point for payday.
This method runs every Thursday. It is structured such that it can be
run again safely (with a newly-instantiated Payday object) if it
crashes.
"""
log("Greetings, program! It's PAYDAY!!!!")
ts_start = self.start()
self.zero_out_pending(ts_start)
def genparticipants(for_payday):
"""Closure generator to yield participants with tips and total.
We re-fetch participants each time, because the second time through
we want to use the total obligations they have for next week, and
if we pass a non-False for_payday to get_tips_and_total then we
only get unfulfilled tips from prior to that timestamp, which is
none of them by definition.
If someone changes tips after payout starts, and we crash during
payout, then their new tips_and_total will be used on the re-run.
That's okay.
Note that we take ts_start from the outer scope when we pass it to
get_participants, but we pass in for_payday, because we might want
it to be False (per the definition of git_tips_and_total).
"""
for participant in self.get_participants(ts_start):
tips, total = get_tips_and_total(participant['id'],
for_payday=for_payday,
db=self.db)
typecheck(total, Decimal)
yield (participant, tips, total)
self.payin(ts_start, genparticipants(ts_start))
self.clear_pending_to_balance()
self.payout(ts_start, genparticipants(False))
self.end()
def payday_one(payday_start, participant):
"""Given one participant record, pay their day.
Charge each participants' credit card if needed before transfering money
between Gittip accounts.
"""
tips, total = get_tips_and_total(participant['id'],
for_payday=payday_start)
typecheck(total, decimal.Decimal)
short = total - participant['balance']
if short > 0:
charge(participant['id'], participant['pmt'], short)
ntips = 0
for tip in tips:
if tip['amount'] == 0:
continue
if not transfer(participant['id'], tip['tippee'], tip['amount']):
# The transfer failed due to a lack of funds for the
# participant. Don't try any further transfers.
log("FAILURE: $%s from %s to %s." %
(tip['amount'], participant['id'], tip['tippee']))
break
log("SUCCESS: $%s from %s to %s." %
(tip['amount'], participant['id'], tip['tippee']))
ntips += 1
# Update stats.
# =============
STATS = """\
UPDATE paydays
SET nparticipants = nparticipants + 1
, ntippers = ntippers + %s
, ntips = ntips + %s
WHERE ts_end='1970-01-01T00:00:00+00'::timestamptz
RETURNING id
"""
assert_one_payday(db.fetchone(STATS, (1 if ntips > 0 else 0, ntips)))
def update_receiving_amounts(self):
UPDATE = """
CREATE OR REPLACE TEMPORARY VIEW total_receiving AS
SELECT tippee, sum(amount) AS amount, count(*) AS ntippers
FROM current_tips
JOIN participants p ON p.username = tipper
WHERE p.is_suspicious IS NOT TRUE
AND p.last_bill_result = ''
AND amount > 0
GROUP BY tippee;
UPDATE participants
SET receiving = (amount + taking)
, npatrons = ntippers
FROM total_receiving
WHERE tippee = username;
"""
with self.db.get_cursor() as cursor:
cursor.execute(UPDATE)
log("Updated receiving amounts.")
def api_get(self, path, sess=None, **kw):
"""
Given a `path` (e.g. /users/foo), this function sends a GET request to
the platform's API (e.g. https://api.github.com/users/foo).
The response is returned, after checking its status code and ratelimit
headers.
"""
is_user_session = bool(sess)
if not sess:
sess = self.get_auth_session()
response = sess.get(self.api_url+path, **kw)
limit, remaining, reset = self.get_ratelimit_headers(response)
if not is_user_session:
self.log_ratelimit_headers(limit, remaining, reset)
# Check response status
status = response.status_code
if status == 401 and isinstance(self, PlatformOAuth1):
# https://tools.ietf.org/html/rfc5849#section-3.2
if is_user_session:
raise TokenExpiredError
raise Response(500)
if status == 404:
raise Response(404, response.text)
if status == 429 and is_user_session:
def msg(_, to_age):
if remaining == 0 and reset:
return _("You've consumed your quota of requests, you can try again in {0}.", to_age(reset))
else:
return _("You're making requests too fast, please try again later.")
raise LazyResponse(status, msg)
if status != 200:
log('{} api responded with {}:\n{}'.format(self.name, status, response.text)
, level=logging.ERROR)
msg = lambda _: _("{0} returned an error, please try again later.",
self.display_name)
raise LazyResponse(502, msg)
return response
def clear_pending_to_balance(self):
"""Transfer pending into balance, setting pending to NULL.
Any users that were created while the payin loop was running will have
pending NULL (the default). If we try to add that to balance we'll get
a NULL (0.0 + NULL = NULL), and balance has a NOT NULL constraint.
Hence the where clause. See:
https://github.com/gittip/www.gittip.com/issues/170
"""
self.db.execute("""\
UPDATE participants
SET balance = (balance + pending)
, pending = NULL
WHERE pending IS NOT NULL
""")
log("Cleared pending to balance. Ready for payouts.")
def check_balances(cursor):
"""Check that balances aren't becoming (more) negative
"""
oops = cursor.one("""
SELECT *
FROM (
SELECT p.id
, p.username
, (p.balance + p2.new_balance - p2.old_balance) AS new_balance
, p.balance AS cur_balance
FROM payday_participants p2
JOIN participants p ON p.id = p2.id
AND p2.new_balance <> p2.old_balance
) foo
WHERE new_balance < 0 AND new_balance < cur_balance
LIMIT 1
""")
if oops:
log(oops)
raise NegativeBalance()
log("Checked the balances.")
def pachinko(self, ts_start, participants):
i = 0
for i, (participant, foo, bar) in enumerate(participants, start=1):
if i % 100 == 0:
log("Pachinko done for %d participants." % i)
if participant.number != 'plural':
continue
available = participant.balance
log("Pachinko out from %s with $%s." %
(participant.username, available))
def tip(member, amount):
tip = {}
tip['tipper'] = participant.username
tip['tippee'] = member['username']
tip['amount'] = amount
tip['claimed_time'] = ts_start
self.tip(participant, tip, ts_start, pachinko=True)
return tip['amount']
for member in participant.get_members():
amount = min(member['take'], available)
available -= amount
tip(member, amount)
if available == 0:
break
log("Did pachinko for %d participants." % i)
def payout(self):
"""This is the second stage of payday in which we send money out to the
bank accounts of participants.
"""
log("Starting payout loop.")
participants = self.db.all("""
SELECT p.*::participants
FROM participants p
WHERE balance > 0
AND ( SELECT count(*)
FROM exchange_routes r
WHERE r.participant = p.id
AND network = 'balanced-ba'
) > 0
""")
def credit(participant):
if participant.is_suspicious is None:
log("UNREVIEWED: %s" % participant.username)
return
withhold = participant.giving + participant.pledging
error = ach_credit(self.db, participant, withhold)
if error:
self.mark_ach_failed()
threaded_map(credit, participants)
log("Did payout for %d participants." % len(participants))
self.db.self_check()
log("Checked the DB.")
def oauth_dance(website, qs):
"""Given a querystring, return a dict of user_info.
The querystring should be the querystring that we get from GitHub when
we send the user to the return value of oauth_url above.
See also:
http://developer.github.com/v3/oauth/
"""
log("Doing an OAuth dance with Github.")
if 'error' in qs:
raise Response(500, str(qs['error']))
data = {
'code': qs['code'].encode('US-ASCII'),
'client_id': website.github_client_id,
'client_secret': website.github_client_secret
}
r = requests.post("https://github.com/login/oauth/access_token",
data=data)
assert r.status_code == 200, (r.status_code, r.text)
back = dict([pair.split('=') for pair in r.text.split('&')]) # XXX
if 'error' in back:
raise Response(400, back['error'].encode('utf-8'))
assert back.get('token_type', '') == 'bearer', back
access_token = back['access_token']
r = requests.get("https://api.github.com/user",
headers={'Authorization': 'token %s' % access_token})
assert r.status_code == 200, (r.status_code, r.text)
user_info = json.loads(r.text)
log("Done with OAuth dance with Github for %s (%s)." %
(user_info['login'], user_info['id']))
return user_info
def payout(self):
"""This is the second stage of payday in which we send money out to the
bank accounts of participants.
"""
log("Starting payout loop.")
participants = self.db.all("""
SELECT p.*::participants
FROM participants p
WHERE balance > 0
AND balanced_customer_href IS NOT NULL
AND last_ach_result IS NOT NULL
""")
def credit(participant):
if participant.is_suspicious is None:
log("UNREVIEWED: %s" % participant.username)
return
withhold = participant.giving + participant.pledging
error = ach_credit(self.db, participant, withhold)
if error:
self.mark_ach_failed()
threaded_map(credit, participants)
log("Did payout for %d participants." % len(participants))
self.db.self_check()
log("Checked the DB.")
def start(cls):
"""Try to start a new Payday.
If there is a Payday that hasn't finished yet, then the UNIQUE
constraint on ts_end will kick in and notify us of that. In that case
we load the existing Payday and work on it some more. We use the start
time of the current Payday to synchronize our work.
"""
try:
d = cls.db.one("""
INSERT INTO paydays DEFAULT VALUES
RETURNING id, (ts_start AT TIME ZONE 'UTC') AS ts_start, stage
""",
back_as=dict)
log("Starting a new payday.")
except IntegrityError: # Collision, we have a Payday already.
d = cls.db.one("""
SELECT id, (ts_start AT TIME ZONE 'UTC') AS ts_start, stage
FROM paydays
WHERE ts_end='1970-01-01T00:00:00+00'::timestamptz
""",
back_as=dict)
log("Picking up with an existing payday.")
d['ts_start'] = d['ts_start'].replace(tzinfo=aspen.utils.utc)
log("Payday started at %s." % d['ts_start'])
payday = Payday()
payday.__dict__.update(d)
return payday
def start(self):
"""Try to start a new Payday.
If there is a Payday that hasn't finished yet, then the UNIQUE
constraint on ts_end will kick in and notify us of that. In that case
we load the existing Payday and work on it some more. We use the start
time of the current Payday to synchronize our work.
"""
try:
ts_start = self.db.one("INSERT INTO paydays DEFAULT VALUES "
"RETURNING ts_start")
log("Starting a new payday.")
except IntegrityError: # Collision, we have a Payday already.
ts_start = self.db.one("""
SELECT ts_start
FROM paydays
WHERE ts_end='1970-01-01T00:00:00+00'::timestamptz
""")
log("Picking up with an existing payday.")
log("Payday started at %s." % ts_start)
return ts_start
def update_balances(cursor):
log("Updating balances.")
participants = cursor.all("""
UPDATE participants p
SET balance = (balance + p2.new_balance - p2.old_balance)
FROM payday_participants p2
WHERE p.id = p2.id
AND p2.new_balance <> p2.old_balance
RETURNING p.id
, p.username
, balance AS new_balance
, ( SELECT balance
FROM participants p3
WHERE p3.id = p.id
) AS cur_balance;
""")
# Check that balances aren't becoming (more) negative
for p in participants:
if p.new_balance < 0 and p.new_balance < p.cur_balance:
log(p)
raise NegativeBalance()
cursor.run("""
INSERT INTO payments (timestamp, participant, team, amount, direction, payday)
SELECT *, (SELECT id FROM paydays WHERE extract(year from ts_end) = 1970)
FROM payday_payments;
""")
log("Updated the balances of %i participants." % len(participants))
def pachinko(self, ts_start, participants):
i = 0
for i, (participant, takes) in enumerate(participants, start=1):
if i % 100 == 0:
log("Pachinko done for %d participants." % i)
available = participant.balance
log("Pachinko out from %s with $%s." %
(participant.username, available))
def tip(tippee, amount):
tip = {}
tip['tipper'] = participant.username
tip['tippee'] = tippee
tip['amount'] = amount
tip['claimed_time'] = ts_start
self.tip(participant, tip, ts_start, pachinko=True)
for take in takes:
amount = min(take['amount'], available)
available -= amount
tip(take['member'], amount)
if available == 0:
break
log("Did pachinko for %d participants." % i)
def payday():
gittip.wire_db()
gittip.wire_samurai()
# Lazily import the billing module.
# =================================
# This dodges a problem where db in billing is None if we import it from
# gittip before calling wire_samurai, and it also dodges:
#
# https://github.com/FeeFighters/samurai-client-python/issues/8
from gittip import billing
try:
billing.payday()
except KeyboardInterrupt:
pass
except:
import aspen
import traceback
aspen.log(traceback.format_exc())
def get_response(self, context):
"""Given a context dict, return a response object.
"""
request = context['request']
# find an Accept header
accept = request.headers.get('X-Aspen-Accept', None)
if accept is not None: # indirect negotiation
failure = Response(404)
else: # direct negotiation
accept = request.headers.get('Accept', None)
msg = "The following media types are available: %s."
msg %= ', '.join(self.available_types)
failure = Response(406, msg.encode('US-ASCII'))
# negotiate or punt
render, media_type = self.pages[2] # default to first content page
if accept is not None:
try:
media_type = mimeparse.best_match(self.available_types, accept)
except:
# exception means don't override the defaults
log("Problem with mimeparse.best_match(%r, %r): %r " % (self.available_types, accept, sys.exc_info()))
else:
if media_type == '': # breakdown in negotiations
raise failure
del failure
render = self.renderers[media_type] # KeyError is a bug
response = context['response']
response.body = render(context)
if 'Content-Type' not in response.headers:
response.headers['Content-Type'] = media_type
if media_type.startswith('text/'):
charset = response.charset
if charset is not None:
response.headers['Content-Type'] += '; charset=' + charset
return response
def create_github_review_issue(self):
"""POST to GitHub, and return the URL of the new issue.
"""
api_url = "https://api.github.com/repos/{}/issues".format(self.review_repo)
data = json.dumps({ "title": self.name
, "body": "https://gratipay.com/{}/\n\n".format(self.slug) +
"(This application will remain open for at least a week.)"
})
out = ''
try:
r = requests.post(api_url, auth=self.review_auth, data=data)
if r.status_code == 201:
out = r.json()['html_url']
else:
log(r.status_code)
log(r.text)
err = str(r.status_code)
except:
err = "eep"
if not out:
out = "https://github.com/gratipay/team-review/issues#error-{}".format(err)
return out
def update_stats(self):
self.db.run(
"""\
WITH our_transfers AS (
SELECT *
FROM transfers
WHERE "timestamp" >= %(ts_start)s
AND status = 'succeeded'
)
, our_tips AS (
SELECT *
FROM our_transfers
WHERE context = 'tip'
)
, our_takes AS (
SELECT *
FROM our_transfers
WHERE context = 'take'
)
UPDATE paydays
SET nactive = (
SELECT DISTINCT count(*) FROM (
SELECT tipper FROM our_transfers
UNION
SELECT tippee FROM our_transfers
) AS foo
)
, ntippers = (SELECT count(DISTINCT tipper) FROM our_transfers)
, ntippees = (SELECT count(DISTINCT tippee) FROM our_transfers)
, ntips = (SELECT count(*) FROM our_tips)
, ntakes = (SELECT count(*) FROM our_takes)
, take_volume = (SELECT COALESCE(sum(amount), 0) FROM our_takes)
, ntransfers = (SELECT count(*) FROM our_transfers)
, transfer_volume = (SELECT COALESCE(sum(amount), 0) FROM our_transfers)
WHERE ts_end='1970-01-01T00:00:00+00'::timestamptz
""", {'ts_start': self.ts_start})
log("Updated payday stats.")
def take_over_balances(self):
"""If an account that receives money is taken over during payin we need
to transfer the balance to the absorbing account.
"""
log("Taking over balances.")
for i in itertools.count():
if i > 10:
raise Exception('possible infinite loop')
count = self.db.one("""
DROP TABLE IF EXISTS temp;
CREATE TEMPORARY TABLE temp AS
SELECT archived_as, absorbed_by, balance AS archived_balance
FROM absorptions a
JOIN participants p ON a.archived_as = p.username
WHERE balance > 0;
SELECT count(*) FROM temp;
""")
if not count:
break
self.db.run("""
INSERT INTO transfers (tipper, tippee, amount, context)
SELECT archived_as, absorbed_by, archived_balance, 'take-over'
FROM temp;
UPDATE participants
SET balance = (balance - archived_balance)
FROM temp
WHERE username = archived_as;
UPDATE participants
SET balance = (balance + archived_balance)
FROM temp
WHERE username = absorbed_by;
""")
def get_user_info(login):
"""Get the given user's information from the DB or failing that, github.
:param login:
A unicode string representing a username in github.
:returns:
A dictionary containing github specific information for the user.
"""
typecheck(login, unicode)
rec = gittip.db.fetchone(
"SELECT user_info FROM elsewhere "
"WHERE platform='github' "
"AND user_info->'login' = %s", (login, ))
if rec is not None:
user_info = rec['user_info']
else:
url = "https://api.github.com/users/%s"
user_info = requests.get(url % login,
params={
'client_id':
os.environ.get('GITHUB_CLIENT_ID'),
'client_secret':
os.environ.get('GITHUB_CLIENT_SECRET')
})
status = user_info.status_code
content = user_info.text
if status == 200:
user_info = json.loads(content)
elif status == 404:
raise Response(404,
"GitHub identity '{0}' not found.".format(login))
else:
log("Github api responded with {0}: {1}".format(status, content),
level=logging.WARNING)
raise Response(502, "GitHub lookup failed with %d." % status)
return user_info