def charge_and_transfer(db, payin, payer, statement_descriptor, on_behalf_of=None): """Create a standalone Charge then multiple Transfers. Doc: https://stripe.com/docs/connect/charges-transfers As of January 2019 this only works if the recipients are in the SEPA. """ assert payer.id == payin.payer amount = payin.amount route = ExchangeRoute.from_id(payer, payin.route) try: charge = stripe.Charge.create( amount=Money_to_int(amount), currency=amount.currency.lower(), customer=route.remote_user_id, metadata={'payin_id': payin.id}, on_behalf_of=on_behalf_of, source=route.address, statement_descriptor=statement_descriptor, expand=['balance_transaction'], idempotency_key='payin_%i' % payin.id, ) except stripe.error.StripeError as e: return update_payin(db, payin.id, '', 'failed', repr_stripe_error(e)) except Exception as e: from liberapay.website import website website.tell_sentry(e, {}) return update_payin(db, payin.id, '', 'failed', str(e)) payin = settle_charge_and_transfers(db, payin, charge) send_payin_notification(payin, payer, charge, route) return payin
def main(override_payday_checks=False): from liberapay.billing.transactions import sync_with_mangopay from liberapay.main import website # https://github.com/liberapay/salon/issues/19#issuecomment-191230689 from liberapay.billing.payday import Payday if not website.env.override_payday_checks and not override_payday_checks: # Check that payday hasn't already been run this week r = website.db.one(""" SELECT id FROM paydays WHERE ts_start >= now() - INTERVAL '6 days' AND ts_end >= ts_start """) assert not r, "payday has already been run this week" # Prevent a race condition, by acquiring a DB lock conn = website.db.get_connection().__enter__() cursor = conn.cursor() lock = cursor.one("SELECT pg_try_advisory_lock(1)") assert lock, "failed to acquire the payday lock" try: sync_with_mangopay(website.db) Payday.start().run(website.env.log_dir, website.env.keep_payday_logs) except KeyboardInterrupt: # pragma: no cover pass except Exception as e: # pragma: no cover website.tell_sentry(e, {}, allow_reraise=False) raise finally: conn.close()
def execute_transfer(db, pt, amount, destination, source_transaction): """Create a Transfer. Args: pt (Record): a row from the `payin_transfers` table amount (Money): the amount of the transfer destination (str): the Stripe ID of the destination account source_transaction (str): the ID of the Charge this transfer is linked to Returns: Record: the row updated in the `payin_transfers` table """ try: tr = stripe.Transfer.create( amount=Money_to_int(amount), currency=amount.currency, destination=destination, metadata={'payin_transfer_id': pt.id}, source_transaction=source_transaction, idempotency_key='payin_transfer_%i' % pt.id, ) except stripe.error.StripeError as e: return update_payin_transfer( db, pt.id, '', 'failed', repr_stripe_error(e), amount=amount ) except Exception as e: website.tell_sentry(e, {}) return update_payin_transfer(db, pt.id, '', 'failed', str(e), amount=amount) # `Transfer` objects don't have a `status` attribute, so if no exception was # raised we assume that the transfer was successful. return update_payin_transfer(db, pt.id, tr.id, 'succeeded', None, amount=amount)
def execute_transfer(db, t_id, tr): try: tr.save() except Exception as e: error = repr_exception(e) _record_transfer_result(db, t_id, 'failed', error) from liberapay.website import website website.tell_sentry(e, {}, allow_reraise=False) raise TransferError(error) return record_transfer_result(db, t_id, tr, _raise=True)
def check_wallet_balance(w, state={}): remote_wallet = Wallet.get(w.remote_id) remote_balance = remote_wallet.balance / 100 try: assert remote_balance == w.balance, ( "balances don't match for user #%s (liberapay id %s), wallet #%s contains %s, we expected %s" % (w.remote_owner_id, w.owner, w.remote_id, remote_balance, w.balance) ) except AssertionError as e: from liberapay.website import website website.tell_sentry(e, state, allow_reraise=False)
def hit_rate_limit(db, key_prefix, key_unique, exception=None): try: cap, period = RATE_LIMITS[key_prefix] key = '%s:%s' % (key_prefix, key_unique) r = db.one("SELECT hit_rate_limit(%s, %s, %s)", (key, cap, period)) except Exception as e: from liberapay.website import website website.tell_sentry(e, {}) return -1 if r is None and exception is not None: raise exception(key_unique) return r
def handle_email_bounces(): """Process SES notifications, fetching them from SQS. """ sqs = boto3.resource('sqs', region_name=website.app_conf.ses_region) ses_queue = sqs.Queue(website.app_conf.ses_feedback_queue_url) while True: messages = ses_queue.receive_messages(WaitTimeSeconds=20, MaxNumberOfMessages=10) if not messages: break for msg in messages: try: _handle_ses_notification(msg) except Exception as e: website.tell_sentry(e, {}) sleep(1)
def handle_email_bounces(): """Process SES notifications, fetching them from SQS. """ sqs = boto3.resource('sqs', region_name=website.app_conf.ses_region) ses_queue = sqs.Queue(website.app_conf.ses_feedback_queue_url) while True: messages = ses_queue.receive_messages(WaitTimeSeconds=20, MaxNumberOfMessages=10) if not messages: break for msg in messages: try: _handle_ses_notification(msg) except Exception as e: website.tell_sentry(e, {}) time.sleep(1)
def n_get_text(state, loc, s, p, n, *a, **kw): escape = state['escape'] n = n or 0 msg = loc.catalog.get((s, p)) s2 = None if msg: try: s2 = msg.string[loc.catalog.plural_func(n)] except Exception as e: website.tell_sentry(e, state) if not s2: loc = LOCALE_EN s2 = s if n == 1 else p kw['n'] = format_number(n, locale=loc) or n if isinstance(s2, bytes): s2 = s2.decode('ascii') return i_format(loc, escape(s2), *a, **kw)
def n_get_text(state, loc, s, p, n, *a, **kw): escape = state['escape'] n = n or 0 msg = loc.catalog.get((s, p)) s2 = None if msg: try: s2 = msg.string[loc.catalog.plural_func(n)] except Exception as e: website.tell_sentry(e, state, allow_reraise=True) if not s2: loc = LOCALE_EN s2 = s if n == 1 else p kw['n'] = format_number(n, locale=loc) or n if isinstance(s2, bytes): s2 = s2.decode('ascii') return i_format(loc, escape(s2), *a, **kw)
def test_email_domain(email: NormalizedEmailAddress): """Attempt to resolve an email domain and connect to one of its SMTP servers. Raises: BrokenEmailDomain: if we're unable to establish an SMTP connection EmailDomainUnresolvable: if the DNS lookups fail NonEmailDomain: if the domain doesn't accept email (RFC 7505) """ start_time = time.monotonic() try: ip_addresses = get_email_server_addresses(email) exceptions = [] n_ip_addresses = 0 n_attempts = 0 success = False for ip_addr in ip_addresses: n_ip_addresses += 1 try: if website.app_conf.check_email_servers: test_email_server(str(ip_addr)) success = True break except (SMTPException, OSError) as e: exceptions.append(e) except Exception as e: website.tell_sentry(e) exceptions.append(e) n_attempts += 1 if n_attempts >= 3: break time_elapsed = time.monotonic() - start_time if time_elapsed >= website.app_conf.socket_timeout: break if not success: if n_ip_addresses == 0: raise EmailDomainUnresolvable( email, ("didn't find any public IP address to deliver emails to")) raise BrokenEmailDomain(email, exceptions[0]) except EmailAddressError: raise except NXDOMAIN: raise EmailDomainUnresolvable(email, "no such domain (NXDOMAIN)") except DNSException as e: raise EmailDomainUnresolvable(email, str(e))
def get_email_server_addresses(email): """Resolve an email domain to IP addresses. Yields `IPv4Address` and `IPv6Address` objects. Raises: NonEmailDomain: if the domain doesn't accept email (RFC 7505) DNSException: if a DNS query fails Spec: https://tools.ietf.org/html/rfc5321#section-5.1 """ domain = email.domain rrset = DNS.query(domain, 'MX', raise_on_no_answer=False).rrset if rrset: if len(rrset) == 1 and str(rrset[0].exchange) == '.': # This domain doesn't accept email. https://tools.ietf.org/html/rfc7505 raise NonEmailDomain( email, f"the domain {domain} has a 'null MX' record (RFC 7505)") # Sort the returned MX records records = sorted(rrset, key=lambda rec: (rec.preference, random())) mx_domains = [str(rec.exchange).rstrip('.') for rec in records] else: mx_domains = [domain] # Yield the IP addresses, in order, without duplicates # We limit ourselves to looking up a maximum of 5 domains exceptions = [] seen = set() for mx_domain in mx_domains[:5]: try: mx_ip_addresses = get_public_ip_addresses(mx_domain) except (DNSException, OSError) as e: exceptions.append(e) continue except Exception as e: website.tell_sentry(e) exceptions.append(e) continue for addr in mx_ip_addresses: if addr not in seen: yield addr seen.add(addr) if exceptions: raise exceptions[0]
def execute_transfer(db, pt, amount, destination, source_transaction): """Create a Transfer. Args: pt (Record): a row from the `payin_transfers` table amount (Money): the amount of the transfer destination (str): the Stripe ID of the destination account source_transaction (str): the ID of the Charge this transfer is linked to Returns: Record: the row updated in the `payin_transfers` table """ try: tr = stripe.Transfer.create( amount=Money_to_int(amount), currency=amount.currency, destination=destination, metadata={'payin_transfer_id': pt.id}, source_transaction=source_transaction, idempotency_key='payin_transfer_%i' % pt.id, ) except stripe.error.StripeError as e: return update_payin_transfer(db, pt.id, '', 'failed', repr_stripe_error(e), amount=amount) except Exception as e: website.tell_sentry(e, {}) return update_payin_transfer(db, pt.id, '', 'failed', str(e), amount=amount) # `Transfer` objects don't have a `status` attribute, so if no exception was # raised we assume that the transfer was successful. return update_payin_transfer(db, pt.id, tr.id, 'succeeded', None, amount=amount)
def get_text(state, loc, s, *a, **kw): escape = state['escape'] msg = loc.catalog.get(s) s2 = None if msg: s2 = msg.string if isinstance(s2, tuple): s2 = s2[0] if not s2: s2 = s loc = LOCALE_EN if a or kw: try: return i_format(loc, escape(_decode(s2)), *a, **kw) except Exception as e: website.tell_sentry(e, state) return i_format(LOCALE_EN, escape(_decode(s)), *a, **kw) return escape(s2)
def normalize_email_address(email): """Normalize an email address. Returns: str: the normalized email address Raises: BadEmailAddress: if the address appears to be invalid BadEmailDomain: if the domain name is invalid """ # Remove any surrounding whitespace email = email.strip() # Split the address try: local_part, domain = email.rsplit('@', 1) except ValueError: raise BadEmailAddress(email) # Lowercase and encode the domain name try: domain = domain.lower().encode('idna').decode() except UnicodeError: raise BadEmailDomain(domain) # Check the syntax and length of the address email = local_part + '@' + domain if not EMAIL_RE.match(email) or len(email) > 320: # The length limit is from https://tools.ietf.org/html/rfc3696#section-3 raise BadEmailAddress(email) # Check that the domain has at least one MX record if website.app_conf.check_email_domains: try: DNS.query(domain, 'MX') except DNSException: raise BadEmailDomain(domain) except Exception as e: website.tell_sentry(e, {}) return email
def get_text(state, loc, s, *a, **kw): escape = state['escape'] msg = loc.catalog.get(s) s2 = None if msg: s2 = msg.string if isinstance(s2, tuple): s2 = s2[0] if not s2: s2 = s if loc != LOCALE_EN: loc = LOCALE_EN state['partial_translation'] = True if a or kw: try: return i_format(loc, escape(_decode(s2)), *a, **kw) except Exception as e: website.tell_sentry(e, state) return i_format(LOCALE_EN, escape(_decode(s)), *a, **kw) return escape(s2)
def destination_charge(db, payin, payer, statement_descriptor): """Create a Destination Charge. Doc: https://stripe.com/docs/connect/destination-charges Destination charges don't have built-in support for processing payments "at cost", so we (mis)use transfer reversals to recover the exact amount of the Stripe fee. """ assert payer.id == payin.payer pt = db.one("SELECT * FROM payin_transfers WHERE payin = %s", (payin.id, )) destination = db.one("SELECT id FROM payment_accounts WHERE pk = %s", (pt.destination, )) amount = payin.amount route = ExchangeRoute.from_id(payer, payin.route) if destination == 'acct_1ChyayFk4eGpfLOC': # Stripe rejects the charge if the destination is our own account destination = None else: destination = {'account': destination} try: charge = stripe.Charge.create( amount=Money_to_int(amount), currency=amount.currency.lower(), customer=route.remote_user_id, destination=destination, metadata={'payin_id': payin.id}, source=route.address, statement_descriptor=statement_descriptor, expand=['balance_transaction'], idempotency_key='payin_%i' % payin.id, ) except stripe.error.StripeError as e: return update_payin(db, payin.id, '', 'failed', repr_stripe_error(e)) except Exception as e: website.tell_sentry(e, {}) return update_payin(db, payin.id, '', 'failed', str(e)) payin = settle_destination_charge(db, payin, charge, pt) send_payin_notification(payin, payer, charge, route) return payin
def destination_charge(db, payin, payer, statement_descriptor): """Create a Destination Charge. Doc: https://stripe.com/docs/connect/destination-charges Destination charges don't have built-in support for processing payments "at cost", so we (mis)use transfer reversals to recover the exact amount of the Stripe fee. """ assert payer.id == payin.payer pt = db.one("SELECT * FROM payin_transfers WHERE payin = %s", (payin.id,)) destination = db.one("SELECT id FROM payment_accounts WHERE pk = %s", (pt.destination,)) amount = payin.amount route = ExchangeRoute.from_id(payer, payin.route) if destination == 'acct_1ChyayFk4eGpfLOC': # Stripe rejects the charge if the destination is our own account destination = None else: destination = {'account': destination} try: charge = stripe.Charge.create( amount=Money_to_int(amount), currency=amount.currency.lower(), customer=route.remote_user_id, destination=destination, metadata={'payin_id': payin.id}, source=route.address, statement_descriptor=statement_descriptor, expand=['balance_transaction'], idempotency_key='payin_%i' % payin.id, ) except stripe.error.StripeError as e: return update_payin(db, payin.id, '', 'failed', repr_stripe_error(e)) except Exception as e: website.tell_sentry(e, {}) return update_payin(db, payin.id, '', 'failed', str(e)) payin = settle_destination_charge(db, payin, charge, pt) send_payin_notification(payin, payer, charge, route) return payin
def n_get_text(state, loc, s, p, n, *a, **kw): escape = state['escape'] n, wrapper = (n.value, n.wrapper) if isinstance(n, Wrap) else (n, None) n = n or 0 msg = loc.catalog.get((s, p) if s else p) s2 = None if msg: try: s2 = msg.string[loc.catalog.plural_func(n)] except Exception as e: website.tell_sentry(e, state) if not s2: loc = LOCALE_EN s2 = s if n == 1 else p kw['n'] = format_number(n, locale=loc) or n if wrapper: kw['n'] = wrapper % kw['n'] try: return i_format(loc, escape(_decode(s2)), *a, **kw) except Exception as e: website.tell_sentry(e, state) return i_format(LOCALE_EN, escape(_decode(s if n == 1 else p)), *a, **kw)
def n_get_text(state, loc, s, p, n, *a, **kw): escape = state['escape'] n, wrapper = (n.value, n.wrapper) if isinstance(n, Wrap) else (n, None) n = n or 0 msg = loc.catalog.get((s, p) if s else p) s2 = None if msg: try: s2 = msg.string[loc.catalog.plural_func(n)] except Exception as e: website.tell_sentry(e, state) if not s2: s2 = s if n == 1 else p if loc != LOCALE_EN: loc = LOCALE_EN state['partial_translation'] = True kw['n'] = format_number(n, locale=loc) or n if wrapper: kw['n'] = wrapper % kw['n'] try: return i_format(loc, escape(_decode(s2)), *a, **kw) except Exception as e: website.tell_sentry(e, state) return i_format(LOCALE_EN, escape(_decode(s if n == 1 else p)), *a, **kw)
def destination_charge(db, payin, payer, statement_descriptor): """Create a Destination Charge. Doc: https://stripe.com/docs/connect/destination-charges Destination charges don't have built-in support for processing payments "at cost", so we (mis)use transfer reversals to recover the exact amount of the Stripe fee. """ assert payer.id == payin.payer pt = db.one("SELECT * FROM payin_transfers WHERE payin = %s", (payin.id, )) destination = db.one("SELECT id FROM payment_accounts WHERE pk = %s", (pt.destination, )) amount = payin.amount route = ExchangeRoute.from_id(payer, payin.route) if destination == 'acct_1ChyayFk4eGpfLOC': # Stripe rejects the charge if the destination is our own account destination = None else: destination = {'account': destination} try: charge = stripe.Charge.create( amount=amount.int().amount, currency=amount.currency.lower(), customer=route.remote_user_id, destination=destination, metadata={'payin_id': payin.id}, source=route.address, statement_descriptor=statement_descriptor, expand=['balance_transaction'], idempotency_key='payin_%i' % payin.id, ) except stripe.error.StripeError as e: return update_payin(db, payin.id, '', 'failed', repr_stripe_error(e)) except Exception as e: from liberapay.website import website website.tell_sentry(e, {}) return update_payin(db, payin.id, '', 'failed', str(e)) bt = charge.balance_transaction amount_settled = Money(bt.amount, bt.currency.upper()) / 100 fee = Money(bt.fee, bt.currency.upper()) / 100 net_amount = amount_settled - fee if destination: tr = stripe.Transfer.retrieve(charge.transfer) reversal = tr.reversals.create( amount=bt.fee, description="Stripe fee", metadata={'payin_id': payin.id}, idempotency_key='payin_fee_%i' % payin.id, ) r = db.one( """ UPDATE payin_transfers SET status = %s , remote_id = %s , amount = %s WHERE payin = %s RETURNING id """, (payin.status, getattr(charge, 'transfer', None), net_amount, payin.id)) assert r, locals() return update_payin(db, payin.id, charge.id, charge.status, repr_charge_error(charge), amount_settled=amount_settled, fee=fee)
def charge_and_transfer(db, payin, payer, statement_descriptor, on_behalf_of=None): """Create a standalone Charge then multiple Transfers. Doc: https://stripe.com/docs/connect/charges-transfers As of January 2019 this only works if the recipients are in the SEPA. """ assert payer.id == payin.payer amount = payin.amount route = ExchangeRoute.from_id(payer, payin.route) intent = None try: if route.address.startswith('pm_'): intent = stripe.PaymentIntent.create( amount=Money_to_int(amount), confirm=True, currency=amount.currency.lower(), customer=route.remote_user_id, metadata={'payin_id': payin.id}, on_behalf_of=on_behalf_of, payment_method=route.address, return_url=payer.url('giving/pay/stripe/%i' % payin.id), statement_descriptor=statement_descriptor, idempotency_key='payin_intent_%i' % payin.id, ) else: charge = stripe.Charge.create( amount=Money_to_int(amount), currency=amount.currency.lower(), customer=route.remote_user_id, metadata={'payin_id': payin.id}, on_behalf_of=on_behalf_of, source=route.address, statement_descriptor=statement_descriptor, expand=['balance_transaction'], idempotency_key='payin_%i' % payin.id, ) except stripe.error.StripeError as e: return update_payin(db, payin.id, '', 'failed', repr_stripe_error(e)) except Exception as e: from liberapay.website import website website.tell_sentry(e, {}) return update_payin(db, payin.id, '', 'failed', str(e)) if intent: if intent.status == 'requires_action': update_payin(db, payin.id, None, 'awaiting_payer_action', None, intent_id=intent.id) raise NextAction(intent) else: charge = intent.charges.data[0] intent_id = getattr(intent, 'id', None) payin = settle_charge_and_transfers(db, payin, charge, intent_id=intent_id) send_payin_notification(payin, payer, charge, route) return payin
def destination_charge(db, payin, payer, statement_descriptor): """Create a Destination Charge. Doc: https://stripe.com/docs/connect/destination-charges Destination charges don't have built-in support for processing payments "at cost", so we (mis)use transfer reversals to recover the exact amount of the Stripe fee. """ assert payer.id == payin.payer pt = db.one("SELECT * FROM payin_transfers WHERE payin = %s", (payin.id, )) destination = db.one("SELECT id FROM payment_accounts WHERE pk = %s", (pt.destination, )) amount = payin.amount route = ExchangeRoute.from_id(payer, payin.route) intent = None if destination == 'acct_1ChyayFk4eGpfLOC': # Stripe rejects the charge if the destination is our own account destination = None try: if route.address.startswith('pm_'): intent = stripe.PaymentIntent.create( amount=Money_to_int(amount), confirm=True, currency=amount.currency.lower(), customer=route.remote_user_id, metadata={'payin_id': payin.id}, on_behalf_of=destination, payment_method=route.address, return_url=payer.url('giving/pay/stripe/%i' % payin.id), statement_descriptor=statement_descriptor, transfer_data={'destination': destination} if destination else None, idempotency_key='payin_intent_%i' % payin.id, ) else: charge = stripe.Charge.create( amount=Money_to_int(amount), currency=amount.currency.lower(), customer=route.remote_user_id, destination={'account': destination} if destination else None, metadata={'payin_id': payin.id}, source=route.address, statement_descriptor=statement_descriptor, expand=['balance_transaction'], idempotency_key='payin_%i' % payin.id, ) except stripe.error.StripeError as e: return update_payin(db, payin.id, '', 'failed', repr_stripe_error(e)) except Exception as e: website.tell_sentry(e, {}) return update_payin(db, payin.id, '', 'failed', str(e)) if intent: if intent.status == 'requires_action': update_payin(db, payin.id, None, 'awaiting_payer_action', None, intent_id=intent.id) raise NextAction(intent) else: charge = intent.charges.data[0] intent_id = getattr(intent, 'id', None) payin = settle_destination_charge(db, payin, charge, pt, intent_id=intent_id) send_payin_notification(payin, payer, charge, route) return payin
def check_email_address(email: NormalizedEmailAddress) -> None: """Check that an email address isn't blacklisted and has a valid domain. Raises: BrokenEmailDomain: if we're unable to establish an SMTP connection EmailAddressIsBlacklisted: if the address is in our blacklist EmailDomainUnresolvable: if the DNS lookups fail EmailDomainIsBlacklisted: if the domain name is in our blacklist NonEmailDomain: if the domain doesn't accept email """ # Check that the address isn't in our blacklist check_email_blacklist(email) # Check that we can send emails to this domain if website.app_conf.check_email_domains: # First, we look in our database for addresses matching this domain and # added in the last two years. If the percentage of verified addresses # is high enough and the percentage of blacklisted addresses is low # enough, then it's reasonable to conclude that this is a valid email # domain. stats = website.db.one( """ SELECT count(DISTINCT lower(e.address)) AS n_addresses , count(1) FILTER (WHERE e.verified) AS n_verified , ( SELECT count(DISTINCT lower(bl.address)) FROM email_blacklist bl WHERE lower(bl.address) LIKE ('%%_@' || %(domain)s) AND (bl.ignore_after IS NULL OR bl.ignore_after > current_timestamp) ) AS n_blacklisted_addresses FROM emails e WHERE e.address LIKE ('%%_@' || %(domain)s) AND e.added_time > (current_timestamp - interval '2 years') """, dict(domain=email.domain)) is_known_good_domain = ( stats.n_addresses > 0 and stats.n_verified / stats.n_addresses > 0.2 and stats.n_blacklisted_addresses / stats.n_addresses < 0.2) if not is_known_good_domain: # Try to resolve the domain and connect to its SMTP server(s). try: test_email_domain(email) except EmailAddressError as e: if isinstance(e, BrokenEmailDomain): global port_25_is_open if port_25_is_open is None: try: test_email_domain( normalize_email_address('*****@*****.**')) except BrokenEmailDomain: port_25_is_open = False except Exception as e: website.tell_sentry(e) else: port_25_is_open = True if port_25_is_open is False: website.tell_sentry(e, allow_reraise=False) return request = website.state.get({}).get('request') if request: bypass_error = request.body.get( 'email.bypass_error') == 'yes' else: bypass_error = False if bypass_error and e.bypass_allowed: if request: website.db.hit_rate_limit('email.bypass_error', request.source, TooManyAttempts) else: raise except Exception as e: website.tell_sentry(e)