def make_participant(self, username, **kw):
platform = kw.pop('elsewhere', 'github')
domain = kw.pop('domain', '')
kw2 = {}
for key in ('last_bill_result', 'balance', 'mangopay_wallet_id'):
if key in kw:
kw2[key] = kw.pop(key)
kind = kw.setdefault('kind', 'individual')
if kind not in ('group', 'community'):
kw.setdefault('password', 'x')
kw.setdefault('session_token', username)
i = next(self.seq)
kw.setdefault('mangopay_user_id', -i)
kw.setdefault('status', 'active')
if username:
kw['username'] = username
if 'join_time' not in kw:
kw['join_time'] = utcnow()
cols, vals = zip(*kw.items())
cols = ', '.join(cols)
placeholders = ', '.join(['%s']*len(vals))
participant = self.db.one("""
INSERT INTO participants
({0})
VALUES ({1})
RETURNING participants.*::participants
""".format(cols, placeholders), vals)
self.db.run("""
INSERT INTO elsewhere
(platform, user_id, user_name, participant, domain)
VALUES (%s,%s,%s,%s,%s)
""", (platform, participant.id, username, participant.id, domain))
if kind not in ('group', 'community') and participant.mangopay_user_id:
wallet_id = kw2.get('mangopay_wallet_id', -participant.id)
zero = ZERO[participant.main_currency]
self.db.run("""
INSERT INTO wallets
(remote_id, balance, owner, remote_owner_id)
VALUES (%s, %s, %s, %s)
""", (wallet_id, zero, participant.id, participant.mangopay_user_id))
if 'email' in kw:
self.db.run("""
INSERT INTO emails
(participant, address, verified, verified_time)
VALUES (%s, %s, true, now())
""", (participant.id, kw['email']))
if 'last_bill_result' in kw2:
ExchangeRoute.insert(
participant, 'mango-cc', '-1', kw2['last_bill_result'],
currency=participant.main_currency
)
if 'balance' in kw2 and kw2['balance'] != 0:
self.make_exchange('mango-cc', kw2['balance'], 0, participant)
return participant
def test_log_in_with_old_session(self):
alice = self.make_participant('alice')
alice.update_session('x', utcnow() - timedelta(days=1))
alice.authenticated = True
cookies = SimpleCookie()
alice.sign_in(cookies)
print(cookies)
self.check_with_about_me('alice', cookies)
def to_age(dt, **kw):
if isinstance(dt, datetime):
delta = Age(dt - utcnow())
else:
delta = Age(dt - date.today())
kw.setdefault('granularity', 'day')
delta.format_args = kw
return delta
def rotate_key(self):
"""Generate a new key and send it to the secrets manager.
"""
keys = b' '.join([Fernet.generate_key()] + self.fernet_keys).decode()
if self.secrets_manager:
self.secrets_manager.update_secret(SecretId='Fernet', SecretString=keys)
else:
keys = utcnow().date().isoformat() + ' ' + keys
print("No secrets manager, updating the key storage is up to you.")
return keys
def to_age(dt, **kw):
kw.setdefault('add_direction', True)
if isinstance(dt, datetime):
delta = Age(dt - utcnow())
elif isinstance(dt, timedelta):
delta = Age(dt)
else:
delta = Age(dt - date.today())
kw.setdefault('granularity', 'day')
delta.format_args = kw
return delta
def rotate_stored_data(self, wait=True):
"""Re-encrypt all the sensitive information stored in our database.
This function is a special kind of "cron job" that returns one of two
constants from the `liberapay.cron` module: `CRON_ENCORE`, indicating
that the function needs to be run again to continue its work, or
`CRON_STOP`, indicating that all the ciphertexts are up-to-date (or that
it isn't time to rotate yet).
Rows are processed in batches of 50. Timestamps are used to keep track of
progress and to avoid overwriting new data with re-encrypted old data.
The update only starts one week after the new key was generated, unless
`wait` is set to `False`. This delay is to "ensure" that the previous
key is no longer being used to encrypt new data.
"""
update_start = self.fernet_rotation_start + self.KEY_ROTATION_DELAY
if wait:
if utcnow().date() < update_start:
return CRON_STOP
with website.db.get_cursor() as cursor:
batch = cursor.all("""
SELECT id, info
FROM identities
WHERE (info).ts <= %s
ORDER BY (info).ts ASC
LIMIT 50
""", (update_start,))
if not batch:
return CRON_STOP
sql = """
UPDATE identities
SET info = ('fernet', %s, current_timestamp)::encrypted
WHERE id = %s
AND (info).ts = %s;
"""
args_list = [
(self.rotate_message(r.info.payload), r.id, r.info.ts)
for r in batch
]
execute_batch(cursor, sql, args_list)
return CRON_ENCORE
def set_cookie(cookies, key, value, expires=None, httponly=True, path='/'):
key = ensure_str(key)
cookies[key] = ensure_str(value)
cookie = cookies[key]
if expires:
if isinstance(expires, timedelta):
expires += utcnow()
if isinstance(expires, datetime):
expires = to_rfc822(expires)
cookie[str('expires')] = ensure_str(expires)
if httponly:
cookie[str('httponly')] = True
if path:
cookie[str('path')] = ensure_str(path)
if website.cookie_domain:
cookie[str('domain')] = ensure_str(website.cookie_domain)
if website.canonical_scheme == 'https':
cookie[str('secure')] = True
def encrypt_dict(self, dic, allow_single_key=False):
"""Serialize and encrypt a dictionary for storage in the database.
Encrypting partially predictable data may help an attacker break the
encryption key, so to make our data less predictable we randomize the
order of the dict's items before serializing it.
For this to be effective the CBOR serializer must not sort the items
again in an attempt to produce Canonical CBOR, so we explicitly pass
`canonical=False` to the `cbor.dumps` function.
In addition, the dict must not contain only one key if that key is
predictable, so a `CryptoWarning` is emitted when `dic` only contains
one key, unless `allow_single_key` is set to `True`.
"""
dic = self.randomize_dict(dic, allow_single_key=allow_single_key)
serialized = cbor.dumps(dic, canonical=False)
encrypted = self.fernet.encrypt(serialized)
return Encrypted(dict(scheme='fernet', payload=encrypted, ts=utcnow()))
def build_s3_object_url(key):
now = utcnow()
timestamp = now.strftime('%Y%m%dT%H%M%SZ')
today = timestamp.split('T', 1)[0]
region = website.app_conf.s3_region
access_key = website.app_conf.s3_public_access_key
endpoint = website.app_conf.s3_endpoint
assert endpoint.startswith('https://')
host = endpoint[8:]
querystring = (
"X-Amz-Algorithm=AWS4-HMAC-SHA256&"
"X-Amz-Credential={access_key}%2F{today}%2F{region}%2Fs3%2Faws4_request&"
"X-Amz-Date={timestamp}&"
"X-Amz-Expires=86400&"
"X-Amz-SignedHeaders=host"
).format(**locals())
canonical_request = (
"GET\n"
"/{key}\n"
"{querystring}\n"
"host:{host}\n"
"\n"
"host\n"
"UNSIGNED-PAYLOAD"
).format(**locals()).encode()
canonical_request_hash = sha256(canonical_request).hexdigest()
string_to_sign = (
"AWS4-HMAC-SHA256\n"
"{timestamp}\n"
"{today}/{region}/s3/aws4_request\n"
"{canonical_request_hash}"
).format(**locals()).encode()
aws4_secret_key = b"AWS4" + website.app_conf.s3_secret_key.encode()
sig_key = hmac.new(aws4_secret_key, today.encode(), sha256).digest()
sig_key = hmac.new(sig_key, region.encode(), sha256).digest()
sig_key = hmac.new(sig_key, b"s3", sha256).digest()
sig_key = hmac.new(sig_key, b"aws4_request", sha256).digest()
signature = hmac.new(sig_key, string_to_sign, sha256).hexdigest()
return endpoint + "/" + key + "?" + querystring + "&X-Amz-Signature=" + signature
def make_connect_token(self):
token = uuid.uuid4().hex
expires = utcnow() + CONNECT_TOKEN_TIMEOUT
return self.save_connect_token(token, expires)
def update_payin(
db, payin_id, remote_id, status, error,
amount_settled=None, fee=None, intent_id=None, refunded_amount=None,
):
"""Update the status and other attributes of a charge.
Args:
payin_id (int): the ID of the charge in our database
remote_id (str): the ID of the charge in the payment processor's database
status (str): the new status of the charge
error (str): if the charge failed, an error message to show to the payer
Returns:
Record: the row updated in the `payins` table
"""
with db.get_cursor() as cursor:
payin = cursor.one("""
UPDATE payins
SET status = %(status)s
, error = %(error)s
, remote_id = coalesce(remote_id, %(remote_id)s)
, amount_settled = coalesce(amount_settled, %(amount_settled)s)
, fee = coalesce(fee, %(fee)s)
, intent_id = coalesce(intent_id, %(intent_id)s)
, refunded_amount = coalesce(%(refunded_amount)s, refunded_amount)
WHERE id = %(payin_id)s
RETURNING *
, (SELECT status FROM payins WHERE id = %(payin_id)s) AS old_status
""", locals())
if not payin:
return
if remote_id and payin.remote_id != remote_id:
raise AssertionError(f"the remote IDs don't match: {payin.remote_id!r} != {remote_id!r}")
if status != payin.old_status:
cursor.run("""
INSERT INTO payin_events
(payin, status, error, timestamp)
VALUES (%s, %s, %s, current_timestamp)
""", (payin_id, status, error))
if status in ('pending', 'succeeded'):
cursor.run("""
UPDATE exchange_routes
SET status = 'consumed'
WHERE id = %s
AND one_off IS TRUE
""", (payin.route,))
# Lock to avoid concurrent updates
cursor.run("SELECT * FROM participants WHERE id = %s FOR UPDATE",
(payin.payer,))
# Update scheduled payins, if appropriate
if status in ('pending', 'succeeded'):
sp = cursor.one("""
SELECT *
FROM scheduled_payins
WHERE payer = %s
AND payin = %s
""", (payin.payer, payin.id))
if not sp:
# Try to find a scheduled renewal that matches this payin.
# It doesn't have to be an exact match.
schedule = cursor.all("""
SELECT *
FROM scheduled_payins
WHERE payer = %s
AND payin IS NULL
AND mtime < %s
""", (payin.payer, payin.ctime))
today = utcnow().date()
schedule.sort(key=lambda sp: abs((sp.execution_date - today).days))
payin_tippees = set(cursor.all("""
SELECT coalesce(team, recipient) AS tippee
FROM payin_transfers
WHERE payer = %s
AND payin = %s
""", (payin.payer, payin.id)))
for sp in schedule:
if any((tr['tippee_id'] in payin_tippees) for tr in sp.transfers):
cursor.run("""
UPDATE scheduled_payins
SET payin = %s
, mtime = current_timestamp
WHERE id = %s
""", (payin.id, sp.id))
break
return payin
AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ BA BB BD BE BF BG BH BI BJ
BL BM BN BO BQ BR BS BT BV BW BY BZ CA CC CD CF CG CH CI CK CL CM CN CO CR
CU CV CW CX CY CZ DE DJ DK DM DO DZ EC EE EG EH ER ES ET FI FJ FK FM FO FR
GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY HK HM HN HR HT HU
ID IE IL IM IN IO IQ IR IS IT JE JM JO JP KE KG KH KI KM KN KP KR KW KY KZ
LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME MF MG MH MK ML MM MN MO MP MQ
MR MS MT MU MV MW MX MY MZ NA NC NE NF NG NI NL NO NP NR NU NZ OM PA PE PF
PG PH PK PL PM PN PR PS PT PW PY QA RE RO RS RU RW SA SB SC SD SE SG SH SI
SJ SK SL SM SN SO SR SS ST SV SX SY SZ TC TD TF TG TH TJ TK TL TM TN TO TR
TT TV TW TZ UA UG UM US UY UZ VA VC VE VG VI VN VU WF WS YE YT ZA ZM ZW
""".split()
COUNTRIES = make_sorted_dict(COUNTRY_CODES, Locale('en').territories)
CURRENCIES_MAP = {}
today = utcnow().date()
for country, currencies in babel.core.get_global('territory_currencies').items():
for currency, start_date, end_date, tender in currencies:
if currency not in CURRENCIES:
continue
if start_date:
start_date = date(*start_date)
if end_date:
end_date = date(*end_date)
if (start_date is None or start_date <= today) and (end_date is None or end_date >= today):
assert country not in CURRENCIES_MAP
CURRENCIES_MAP[country] = currency
del today
LANGUAGE_CODES_2 = """
aa af ak am ar as az be bg bm bn bo br bs ca cs cy da de dz ee el en eo es
def update_payin(
db,
payin_id,
remote_id,
status,
error,
amount_settled=None,
fee=None,
intent_id=None,
refunded_amount=None,
):
"""Update the status and other attributes of a charge.
Args:
payin_id (int): the ID of the charge in our database
remote_id (str): the ID of the charge in the payment processor's database
status (str): the new status of the charge
error (str): if the charge failed, an error message to show to the payer
Returns:
Record: the row updated in the `payins` table
"""
with db.get_cursor() as cursor:
payin = cursor.one(
"""
WITH old AS (
SELECT * FROM payins WHERE id = %(payin_id)s
)
UPDATE payins
SET status = %(status)s
, error = %(error)s
, remote_id = coalesce(%(remote_id)s, remote_id)
, amount_settled = COALESCE(%(amount_settled)s, amount_settled)
, fee = COALESCE(%(fee)s, fee)
, intent_id = coalesce(%(intent_id)s, intent_id)
, refunded_amount = coalesce(%(refunded_amount)s, refunded_amount)
WHERE id = %(payin_id)s
AND ( status <> %(status)s OR
coalesce_currency_amount(%(refunded_amount)s, amount::currency) <>
coalesce_currency_amount(refunded_amount, amount::currency)
)
RETURNING *
, (SELECT status FROM old) AS old_status
""", locals())
if not payin:
return cursor.one("SELECT * FROM payins WHERE id = %s",
(payin_id, ))
if payin.status != payin.old_status:
cursor.run(
"""
INSERT INTO payin_events
(payin, status, error, timestamp)
VALUES (%s, %s, %s, current_timestamp)
""", (payin_id, status, error))
if payin.status in ('pending', 'succeeded'):
cursor.run(
"""
UPDATE exchange_routes
SET status = 'consumed'
WHERE id = %s
AND one_off IS TRUE
""", (payin.route, ))
# Lock to avoid concurrent updates
cursor.run("SELECT * FROM participants WHERE id = %s FOR UPDATE",
(payin.payer, ))
# Update scheduled payins, if appropriate
if payin.status in ('pending', 'succeeded'):
sp = cursor.one(
"""
SELECT *
FROM scheduled_payins
WHERE payer = %s
AND payin = %s
""", (payin.payer, payin.id))
if not sp:
schedule = cursor.all(
"""
SELECT *
FROM scheduled_payins
WHERE payer = %s
AND payin IS NULL
""", (payin.payer, ))
today = utcnow().date()
schedule.sort(
key=lambda sp: abs((sp.execution_date - today).days))
payin_tippees = set(
cursor.all(
"""
SELECT coalesce(team, recipient) AS tippee
FROM payin_transfers
WHERE payer = %s
AND payin = %s
""", (payin.payer, payin.id)))
for sp in schedule:
matching_tippees_count = 0
other_transfers = []
for tr in sp.transfers:
if tr['tippee_id'] in payin_tippees:
matching_tippees_count += 1
else:
other_transfers.append(tr)
if matching_tippees_count > 0:
if other_transfers:
cursor.run(
"""
UPDATE scheduled_payins
SET payin = %s
, mtime = current_timestamp
WHERE id = %s
""", (payin.id, sp.id))
other_transfers_sum = Money.sum(
(Money(**tr['amount'])
for tr in other_transfers), sp['amount'].
currency) if sp['amount'] else None
cursor.run(
"""
INSERT INTO scheduled_payins
(ctime, mtime, execution_date, payer,
amount, transfers, automatic,
notifs_count, last_notif_ts,
customized, payin)
VALUES (%(ctime)s, now(), %(execution_date)s, %(payer)s,
%(amount)s, %(transfers)s, %(automatic)s,
%(notifs_count)s, %(last_notif_ts)s,
%(customized)s, NULL)
""",
dict(
sp._asdict(),
amount=other_transfers_sum,
transfers=json.dumps(other_transfers),
))
else:
cursor.run(
"""
UPDATE scheduled_payins
SET payin = %s
, mtime = current_timestamp
WHERE id = %s
""", (payin.id, sp.id))
break
elif payin.status == 'failed':
cursor.run(
"""
UPDATE scheduled_payins
SET payin = NULL
WHERE payer = %s
AND payin = %s
""", (payin.payer, payin.id))
return payin
def to_age(dt):
if isinstance(dt, datetime):
return dt - utcnow()
return datedelta(dt - date.today())
def get_start_of_current_utc_day():
"""Returns a `datetime` for the start of the current day in the UTC timezone.
"""
now = utcnow()
return datetime(now.year, now.month, now.day, tzinfo=utc)
def make_participant(self, username, **kw):
platform = kw.pop('elsewhere', 'github')
domain = kw.pop('domain', '')
kw2 = {}
for key in ('route_status', 'balance', 'mangopay_wallet_id'):
if key in kw:
kw2[key] = kw.pop(key)
kind = kw.setdefault('kind', 'individual')
is_person = kind not in ('group', 'community')
if is_person:
i = next(self.seq)
kw.setdefault('mangopay_user_id', -i)
kw.setdefault('status', 'active')
if username:
kw['username'] = username
if 'join_time' not in kw:
kw['join_time'] = utcnow()
kw.setdefault('email_lang', 'en')
kw.setdefault('main_currency', 'EUR')
kw.setdefault('accepted_currencies', kw['main_currency'])
cols, vals = zip(*kw.items())
cols = ', '.join(cols)
placeholders = ', '.join(['%s']*len(vals))
participant = self.db.one("""
INSERT INTO participants
({0})
VALUES ({1})
RETURNING participants.*::participants
""".format(cols, placeholders), vals)
self.db.run("""
INSERT INTO elsewhere
(platform, user_id, user_name, participant, domain)
VALUES (%s,%s,%s,%s,%s)
""", (platform, participant.id, username, participant.id, domain))
if is_person and participant.mangopay_user_id:
wallet_id = kw2.get('mangopay_wallet_id', -participant.id)
zero = ZERO[participant.main_currency]
self.db.run("""
INSERT INTO wallets
(remote_id, balance, owner, remote_owner_id)
VALUES (%s, %s, %s, %s)
""", (wallet_id, zero, participant.id, participant.mangopay_user_id))
if 'email' in kw:
self.db.run("""
INSERT INTO emails
(participant, address, verified, verified_time)
VALUES (%s, %s, true, now())
""", (participant.id, kw['email']))
if 'route_status' in kw2:
ExchangeRoute.insert(
participant, 'mango-cc', '-1', kw2['route_status'],
currency=participant.main_currency
)
if 'balance' in kw2 and kw2['balance'] != 0:
self.make_exchange('mango-cc', kw2['balance'], 0, participant)
return participant
def upsert(cls, i):
"""Insert or update a user's info.
"""
i.info_fetched_at = utcnow()
# Clean up avatar_url
if i.avatar_url:
scheme, netloc, path, query, fragment = urlsplit(i.avatar_url)
fragment = ''
if netloc.endswith('githubusercontent.com') or \
netloc.endswith('gravatar.com') or \
netloc.endswith('libravatar.org'):
query = AVATAR_QUERY
i.avatar_url = urlunsplit((scheme, netloc, path, query, fragment))
# Serialize extra_info
if isinstance(i.extra_info, ET.Element):
i.extra_info = xmltodict.parse(ET.tostring(i.extra_info))
i.extra_info = json.dumps(i.extra_info)
d = dict(i.__dict__)
d.pop('email', None)
cols, vals = zip(*d.items())
cols = ', '.join(cols)
placeholders = ', '.join(['%s'] * len(vals))
def update():
return cls.db.one(
"""
UPDATE elsewhere
SET ({0}) = ({1})
WHERE platform=%s AND domain=%s AND user_id=%s
RETURNING elsewhere.*::elsewhere_with_participant
""".format(cols, placeholders),
vals + (i.platform, i.domain, i.user_id))
account = update() if i.user_id else None
if not account:
try:
# Try to insert the account
# We do this with a transaction so that if the insert fails, the
# participant we reserved for them is rolled back as well.
with cls.db.get_cursor() as cursor:
account = cursor.one(
"""
WITH p AS (
INSERT INTO participants DEFAULT VALUES RETURNING id
)
INSERT INTO elsewhere
(participant, {0})
VALUES ((SELECT id FROM p), {1})
RETURNING elsewhere.*::elsewhere_with_participant
""".format(cols, placeholders), vals)
except IntegrityError:
# The account is already in the DB, update it instead
if i.user_name and i.user_id:
# Set user_id if it was missing
cls.db.run(
"""
UPDATE elsewhere
SET user_id = %s
WHERE platform=%s AND domain=%s AND lower(user_name)=%s
AND user_id IS NULL
""",
(i.user_id, i.platform, i.domain, i.user_name.lower()))
elif not i.user_id:
return cls._from_thing('user_name', i.platform,
i.user_name, i.domain)
account = update()
if not account:
raise
# Return account after propagating avatar_url to participant
account.participant.update_avatar()
return account
def make_connect_token(self):
token = uuid.uuid4().hex
expires = utcnow() + CONNECT_TOKEN_TIMEOUT
return self.save_connect_token(token, expires)
AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ BA BB BD BE BF BG BH BI BJ
BL BM BN BO BQ BR BS BT BV BW BY BZ CA CC CD CF CG CH CI CK CL CM CN CO CR
CU CV CW CX CY CZ DE DJ DK DM DO DZ EC EE EG EH ER ES ET FI FJ FK FM FO FR
GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY HK HM HN HR HT HU
ID IE IL IM IN IO IQ IR IS IT JE JM JO JP KE KG KH KI KM KN KP KR KW KY KZ
LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME MF MG MH MK ML MM MN MO MP MQ
MR MS MT MU MV MW MX MY MZ NA NC NE NF NG NI NL NO NP NR NU NZ OM PA PE PF
PG PH PK PL PM PN PR PS PT PW PY QA RE RO RS RU RW SA SB SC SD SE SG SH SI
SJ SK SL SM SN SO SR SS ST SV SX SY SZ TC TD TF TG TH TJ TK TL TM TN TO TR
TT TV TW TZ UA UG UM US UY UZ VA VC VE VG VI VN VU WF WS YE YT ZA ZM ZW
""".split()
COUNTRIES = make_sorted_dict(COUNTRY_CODES, Locale('en').territories)
CURRENCIES_MAP = {}
today = utcnow().date()
for country, currencies in babel.core.get_global('territory_currencies').items():
for currency, start_date, end_date, tender in currencies:
if currency not in CURRENCIES:
continue
if start_date:
start_date = date(*start_date)
if end_date:
end_date = date(*end_date)
if (start_date is None or start_date <= today) and (end_date is None or end_date >= today):
assert country not in CURRENCIES_MAP
CURRENCIES_MAP[country] = currency
del today
LANGUAGE_CODES_2 = """
aa af ak am ar as az be bg bm bn bo br bs ca cs cy da de dz ee el en eo es
def check_connect_token(self, token):
return (self.connect_token
and constant_time_compare(self.connect_token, token)
and self.connect_expires > utcnow())
def upsert(cls, i):
"""Insert or update a user's info.
"""
i.info_fetched_at = utcnow()
# Clean up avatar_url
if i.avatar_url:
scheme, netloc, path, query, fragment = urlsplit(i.avatar_url)
fragment = ''
if netloc.endswith('githubusercontent.com') or \
netloc.endswith('gravatar.com') or \
netloc.endswith('libravatar.org'):
query = AVATAR_QUERY
i.avatar_url = urlunsplit((scheme, netloc, path, query, fragment))
d = dict(i.__dict__)
d.pop('email', None)
cols, vals = zip(*d.items())
cols = ', '.join(cols)
placeholders = ', '.join(['%s'] * len(vals))
def update():
return cls.db.one(
"""
UPDATE elsewhere
SET ({0}) = ({1})
WHERE platform=%s AND domain=%s AND user_id=%s
RETURNING elsewhere.*::elsewhere_with_participant
""".format(cols, placeholders),
vals + (i.platform, i.domain, i.user_id))
# Check for and handle a possible user_name reallocation
if i.user_name:
conflicts_with = cls.db.one(
"""
SELECT e.*::elsewhere_with_participant
FROM elsewhere e
WHERE e.platform = %s
AND e.domain = %s
AND lower(e.user_name) = %s
AND e.user_id <> %s
""", (i.platform, i.domain, i.user_name.lower(), i.user_id))
if conflicts_with is not None:
try:
conflicts_with.refresh_user_info()
except (UnableToRefreshAccount, UserNotFound):
cls.db.run(
"""
UPDATE elsewhere
SET user_name = null
WHERE id = %s
AND platform = %s
AND domain = %s
AND user_name = %s
""", (conflicts_with.id, i.platform, i.domain,
conflicts_with.user_name))
del conflicts_with
account = update() if i.user_id else None
if not account:
try:
# Try to insert the account
# We do this with a transaction so that if the insert fails, the
# participant we reserved for them is rolled back as well.
with cls.db.get_cursor() as cursor:
account = cursor.one(
"""
WITH p AS (
INSERT INTO participants DEFAULT VALUES RETURNING id
)
INSERT INTO elsewhere
(participant, {0})
VALUES ((SELECT id FROM p), {1})
RETURNING elsewhere.*::elsewhere_with_participant
""".format(cols, placeholders), vals)
except IntegrityError:
# The account is already in the DB, update it instead
if i.user_name and i.user_id:
# Set user_id if it was missing
cls.db.run(
"""
UPDATE elsewhere
SET user_id = %s
WHERE platform=%s AND domain=%s AND lower(user_name)=%s
AND user_id IS NULL
""",
(i.user_id, i.platform, i.domain, i.user_name.lower()))
elif not i.user_id:
return cls._from_thing('user_name', i.platform,
i.user_name, i.domain)
account = update()
if not account:
raise
# Return account after propagating avatar_url to participant
avatar_url = account.avatar_url
if avatar_url and avatar_url.startswith('https://pbs.twimg.com/'):
avatar_url = 'https://nitter.net/pic/' + avatar_url[22:].replace(
'/', '%2F')
account.participant.update_avatar(check=False)
return account
def _handle_ses_notification(msg):
# Doc: https://docs.aws.amazon.com/ses/latest/DeveloperGuide/notification-contents.html
data = json.loads(json.loads(msg.body)['Message'])
notif_type = data['notificationType']
transient = False
if notif_type == 'Bounce':
bounce = data['bounce']
report_id = bounce['feedbackId']
recipients = bounce['bouncedRecipients']
if bounce.get('bounceType') == 'Transient':
transient = True
bounce_subtype = bounce.get('bounceSubType')
if bounce_subtype not in ('General', 'MailboxFull'):
website.warning("unhandled bounce subtype: %r" %
bounce_subtype)
elif notif_type == 'Complaint':
complaint = data['complaint']
report_id = complaint['feedbackId']
recipients = complaint['complainedRecipients']
complaint_type = complaint.get('complaintFeedbackType')
if complaint.get('complaintSubType') == 'OnAccountSuppressionList':
pass
elif complaint_type is None:
# This complaint is invalid, ignore it.
logging.info(
"Received an invalid email complaint without a Feedback-Type. ID: %s"
% report_id)
msg.delete()
return
elif complaint_type not in ('abuse', 'fraud'):
# We'll figure out how to deal with that when it happens.
raise ValueError(complaint_type)
else:
raise ValueError(notif_type)
for recipient in recipients:
address = recipient['emailAddress']
if address[-1] == '>':
address = address[:-1].rsplit('<', 1)[1]
if notif_type == 'Bounce':
# Check the reported delivery status
# Spec: https://tools.ietf.org/html/rfc3464#section-2.3.3
action = recipient.get('action')
if action is None:
# This isn't a standard bounce. It may be a misdirected automatic reply.
continue
elif action == 'failed':
# This is the kind of DSN we're interested in.
pass
elif action == 'delivered':
# The reporting MTA claims that the message has been successfully delivered.
continue
elif action in ('delayed', 'relayed', 'expanded'):
# Ignore non-final DSNs.
continue
else:
# This is a new or non-standard type of DSN, ignore it.
continue
# Check for recurrent "transient" errors
if transient:
ignore_after = utcnow() + timedelta(days=5)
n_previous_bounces = website.db.one(
"""
SELECT count(*)
FROM email_blacklist
WHERE lower(address) = lower(%s)
AND ts > (current_timestamp - interval '90 days')
AND reason = 'bounce'
""", (address, ))
if n_previous_bounces >= 2:
ignore_after = utcnow() + timedelta(days=180)
else:
ignore_after = None
# Add the address to our blacklist
r = website.db.one(
"""
INSERT INTO email_blacklist
(address, reason, ses_data, report_id, ignore_after)
VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (report_id, address) DO NOTHING
RETURNING *
""", (address, notif_type.lower(), json.dumps(data), report_id,
ignore_after))
if r is None:
# Already done
continue
# Attempt to notify the user(s)
bounce_message = get_bounce_message(r.reason, r.ses_data, r.details)
participants = website.db.all(
"""
SELECT p
FROM emails e
JOIN participants p ON p.id = e.participant
WHERE lower(e.address) = lower(%s)
AND (p.email IS NULL OR lower(p.email) = lower(e.address))
""", (address, ))
for p in participants:
try:
p.notify(
'email_blacklisted',
email=False,
web=True,
type='warning',
blacklisted_address=address,
reason=r.reason,
ignore_after=r.ignore_after,
bounce_message=bounce_message,
)
except DuplicateNotification:
continue
msg.delete()
def record_payin_refund(
db,
payin_id,
remote_id,
amount,
reason,
description,
status,
error=None,
ctime=None,
):
"""Record a charge refund.
Args:
payin_id (int): the ID of the refunded payin in our database
remote_id (int): the ID of the refund in the payment processor's database
amount (Money): the refund amount, must be less or equal to the payin amount
reason (str): why this refund was initiated (`refund_reason` SQL type)
description (str): details of the circumstances of this refund
status (str): the current status of the refund (`refund_status` SQL type)
error (str): error message, if the refund has failed
ctime (datetime): when the refund was initiated
Returns:
Record: the row inserted in the `payin_refunds` table
"""
refund = db.one(
"""
INSERT INTO payin_refunds
(payin, remote_id, amount, reason, description,
status, error, ctime)
VALUES (%(payin_id)s, %(remote_id)s, %(amount)s, %(reason)s, %(description)s,
%(status)s, %(error)s, coalesce(%(ctime)s, current_timestamp))
ON CONFLICT (payin, remote_id) DO UPDATE
SET amount = excluded.amount
, reason = excluded.reason
, description = excluded.description
, status = excluded.status
, error = excluded.error
RETURNING *
, ( SELECT old.status
FROM payin_refunds old
WHERE old.payin = %(payin_id)s
AND old.remote_id = %(remote_id)s
) AS old_status
""", locals())
notify = (refund.status in ('pending', 'succeeded')
and refund.status != refund.old_status and refund.ctime >
(utcnow() - timedelta(hours=24)))
if notify:
payin = db.one("SELECT * FROM payins WHERE id = %s", (refund.payin, ))
payer = db.Participant.from_id(payin.payer)
payer.notify(
'payin_refund_initiated',
payin_amount=payin.amount,
payin_ctime=payin.ctime,
refund_amount=refund.amount,
refund_reason=refund.reason,
)
return refund
def check_connect_token(self, token):
return (
self.connect_token and
constant_time_compare(self.connect_token, token) and
self.connect_expires > utcnow()
)
