def test_rate_limited_forward_phase_for_alias(flask_client): user = User.create(email="[email protected]", password="******", name="Test User", activated=True) db.session.commit() # no rate limiting for a new alias alias = Alias.create_new_random(user) db.session.commit() assert not rate_limited_for_alias(alias) # rate limit when there's a previous activity on alias contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", ) db.session.commit() for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_ALIAS + 1): EmailLog.create(user_id=user.id, contact_id=contact.id) db.session.commit() assert rate_limited_for_alias(alias)
def test_rate_limited_reply_phase(flask_client): # no rate limiting when reply_email does not exist assert not rate_limited_reply_phase("*****@*****.**") user = User.create(email="[email protected]", password="******", name="Test User", activated=True) db.session.commit() alias = Alias.create_new_random(user) db.session.commit() contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", ) db.session.commit() for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_ALIAS + 1): EmailLog.create(user_id=user.id, contact_id=contact.id) db.session.commit() assert rate_limited_reply_phase("*****@*****.**")
def test_should_disable_bounces_account(flask_client): """if an account has more than 10 bounces every day for at least 5 days in the last 10 days, disable alias""" user = login(flask_client) alias = Alias.create_new_random(user) Session.commit() # create a lot of bounces on alias contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", commit=True, ) for day in range(6): for _ in range(10): EmailLog.create( user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id, commit=True, bounced=True, created_at=arrow.now().shift(days=-day), ) alias2 = Alias.create_new_random(user) assert should_disable(alias2)
def test_should_disable_bounces_every_day(flask_client): """if an alias has bounces every day at least 9 days in the last 10 days, disable alias""" user = login(flask_client) alias = Alias.create_new_random(user) Session.commit() assert not should_disable(alias) # create a lot of bounce on this alias contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", commit=True, ) for i in range(9): EmailLog.create( user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id, commit=True, bounced=True, created_at=arrow.now().shift(days=-i), ) assert should_disable(alias)
def test_alias_activities(flask_client): user = User.create(email="[email protected]", password="******", name="Test User", activated=True) Session.commit() # create api_key api_key = ApiKey.create(user.id, "for test") Session.commit() alias = Alias.create_new_random(user) Session.commit() # create some alias log contact = Contact.create( website_email="*****@*****.**", reply_email="[email protected]", alias_id=alias.id, user_id=alias.user_id, ) Session.commit() for _ in range(int(PAGE_LIMIT / 2)): EmailLog.create( contact_id=contact.id, is_reply=True, user_id=contact.user_id, alias_id=contact.alias_id, ) for _ in range(int(PAGE_LIMIT / 2) + 2): EmailLog.create( contact_id=contact.id, blocked=True, user_id=contact.user_id, alias_id=contact.alias_id, ) r = flask_client.get( url_for("api.get_alias_activities", alias_id=alias.id, page_id=0), headers={"Authentication": api_key.code}, ) assert r.status_code == 200 assert len(r.json["activities"]) == PAGE_LIMIT for ac in r.json["activities"]: assert ac["from"] assert ac["to"] assert ac["timestamp"] assert ac["action"] assert ac["reverse_alias"] assert ac["reverse_alias_address"] # second page, should return 1 or 2 results only r = flask_client.get( url_for("api.get_alias_activities", alias_id=alias.id, page_id=1), headers={"Authentication": api_key.code}, ) assert len(r.json["activities"]) < 3
def test_should_disable_bounce_consecutive_days(flask_client): user = login(flask_client) alias = Alias.create_new_random(user) Session.commit() contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", commit=True, ) # create 6 bounce on this alias in the last 24h: alias is not disabled for _ in range(6): EmailLog.create( user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id, commit=True, bounced=True, ) assert not should_disable(alias) # create 2 bounces in the last 7 days: alias should be disabled for _ in range(2): EmailLog.create( user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id, commit=True, bounced=True, created_at=arrow.now().shift(days=-3), ) assert should_disable(alias)
def test_should_disable(flask_client): user = User.create( email="[email protected]", password="******", name="Test User", activated=True, include_sender_in_reverse_alias=True, ) alias = Alias.create_new_random(user) db.session.commit() assert not should_disable(alias) # create a lot of bounce on this alias contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", commit=True, ) for _ in range(20): EmailLog.create(user_id=user.id, contact_id=contact.id, commit=True, bounced=True) assert should_disable(alias) # should not affect another alias alias2 = Alias.create_new_random(user) db.session.commit() assert not should_disable(alias2)
def test_greylisting_needed_forward_phase_for_mailbox(flask_client): user = User.create( email="[email protected]", password="******", name="Test User", activated=True ) db.session.commit() alias = Alias.create_new_random(user) db.session.commit() contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", ) db.session.commit() for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_MAILBOX + 1): EmailLog.create(user_id=user.id, contact_id=contact.id) db.session.commit() EmailLog.create(user_id=user.id, contact_id=contact.id) # Create another alias with the same mailbox # will be greylisted as there's a previous activity on mailbox alias2 = Alias.create_new_random(user) db.session.commit() assert greylisting_needed_for_mailbox(alias2)
def test_alias_contacts(flask_client): user = User.create( email="[email protected]", password="******", name="Test User", activated=True ) db.session.commit() # create api_key api_key = ApiKey.create(user.id, "for test") db.session.commit() alias = Alias.create_new_random(user) db.session.commit() # create some alias log for i in range(PAGE_LIMIT + 1): contact = Contact.create( website_email=f"marketing-{i}@example.com", reply_email=f"reply-{i}@a.b", alias_id=alias.id, user_id=alias.user_id, ) db.session.commit() EmailLog.create( contact_id=contact.id, is_reply=True, user_id=contact.user_id, alias_id=contact.alias_id, ) db.session.commit() r = flask_client.get( url_for("api.get_alias_contacts_route", alias_id=alias.id, page_id=0), headers={"Authentication": api_key.code}, ) assert r.status_code == 200 assert len(r.json["contacts"]) == PAGE_LIMIT for ac in r.json["contacts"]: assert ac["creation_date"] assert ac["creation_timestamp"] assert ac["last_email_sent_date"] assert ac["last_email_sent_timestamp"] assert ac["contact"] assert ac["reverse_alias"] assert ac["reverse_alias_address"] # second page, should return 1 result only r = flask_client.get( url_for("api.get_alias_contacts_route", alias_id=alias.id, page_id=1), headers={"Authentication": api_key.code}, ) assert len(r.json["contacts"]) == 1
async def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]: """return whether an email has been delivered and the smtp status ("250 Message accepted", "550 Non-existent email address", etc) """ address = rcpt_to.lower().strip() # alias@SL alias = Alias.get_by(email=address) if not alias: LOG.d("alias %s not exist. Try to see if it can be created on the fly", address) alias = try_auto_create(address) if not alias: LOG.d("alias %s cannot be created on-the-fly, return 550", address) return [(False, "550 SL E3 Email not exist")] mail_from = envelope.mail_from.lower().strip() for mb in alias.mailboxes: # email send from a mailbox to alias if mb.email.lower().strip() == mail_from: LOG.exception("cycle email sent from %s to %s", mb, alias) handle_email_sent_to_ourself(alias, mb, msg, alias.user) return [(True, "250 Message accepted for delivery")] contact = get_or_create_contact(msg["From"], envelope.mail_from, alias) email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id) db.session.commit() if not alias.enabled: LOG.d("%s is disabled, do not forward", alias) email_log.blocked = True db.session.commit() # do not return 5** to allow user to receive emails later when alias is enabled return [(True, "250 Message accepted for delivery")] user = alias.user ret = [] mailboxes = alias.mailboxes # no need to create a copy of message if len(mailboxes) == 1: mailbox = mailboxes[0] ret.append(await forward_email_to_mailbox(alias, msg, email_log, contact, envelope, smtp, mailbox, user)) # create a copy of message for each forward else: for mailbox in mailboxes: ret.append(await forward_email_to_mailbox(alias, copy(msg), email_log, contact, envelope, smtp, mailbox, user)) return ret
def test_alias_contacts(flask_client): user = login(flask_client) alias = Alias.create_new_random(user) Session.commit() # create some alias log for i in range(PAGE_LIMIT + 1): contact = Contact.create( website_email=f"marketing-{i}@example.com", reply_email=f"reply-{i}@a.b", alias_id=alias.id, user_id=alias.user_id, ) Session.commit() EmailLog.create( contact_id=contact.id, is_reply=True, user_id=contact.user_id, alias_id=contact.alias_id, ) Session.commit() r = flask_client.get(f"/api/aliases/{alias.id}/contacts?page_id=0") assert r.status_code == 200 assert len(r.json["contacts"]) == PAGE_LIMIT for ac in r.json["contacts"]: assert ac["creation_date"] assert ac["creation_timestamp"] assert ac["last_email_sent_date"] assert ac["last_email_sent_timestamp"] assert ac["contact"] assert ac["reverse_alias"] assert ac["reverse_alias_address"] assert "block_forward" in ac # second page, should return 1 result only r = flask_client.get(f"/api/aliases/{alias.id}/contacts?page_id=1") assert len(r.json["contacts"]) == 1
def post(self): data = request.get_json() now = datetime.datetime.now() if data.get('eta') is not None: send_at = datetime.datetime.strptime(data['eta'], '%Y-%m-%d %H:%M:%S') if send_at < now: raise ScheduleTimeException( 'schedule time:{} error'.format(send_at)) else: countdown = data['countdown'] send_at = datetime.datetime.now() + datetime.timedelta(**countdown) email = EmailLog.create(subject=data['subject'], body=data['body'], to=data['to'], send_at=send_at) if send_at - now <= datetime.timedelta(minutes=15): send_email.apply_async(args=(email.id, ), eta=send_at) return jsonify()
def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]: """return whether an email has been delivered and the smtp status ("250 Message accepted", "550 Non-existent email address", etc) """ address = rcpt_to.lower().strip() # alias@SL alias = Alias.get_by(email=address) if not alias: LOG.d("alias %s not exist. Try to see if it can be created on the fly", address) alias = try_auto_create(address) if not alias: LOG.d("alias %s cannot be created on-the-fly, return 550", address) return [(False, "550 SL E3")] contact = get_or_create_contact(msg["From"], envelope.mail_from, alias) email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id) if not alias.enabled: LOG.d("%s is disabled, do not forward", alias) email_log.blocked = True db.session.commit() # do not return 5** to allow user to receive emails later when alias is enabled return [(True, "250 Message accepted for delivery")] user = alias.user ret = [] for mailbox in alias.mailboxes: ret.append( forward_email_to_mailbox(alias, msg, email_log, contact, envelope, smtp, mailbox, user)) return ret
def fake_data(): LOG.d("create fake data") # Remove db if exist if os.path.exists("db.sqlite"): LOG.d("remove existing db file") os.remove("db.sqlite") # Create all tables db.create_all() # Create a user user = User.create( email="*****@*****.**", name="John Wick", password="******", activated=True, is_admin=True, enable_otp=False, otp_secret="base32secret3232", intro_shown=True, fido_uuid=None, ) user.include_sender_in_reverse_alias = None user.trial_end = None db.session.commit() # add a profile picture file_path = "profile_pic.svg" s3.upload_from_bytesio( file_path, open(os.path.join(ROOT_DIR, "static", "default-icon.svg"), "rb"), content_type="image/svg", ) file = File.create(user_id=user.id, path=file_path, commit=True) user.profile_picture_id = file.id db.session.commit() # create a bounced email alias = Alias.create_new_random(user) db.session.commit() bounce_email_file_path = "bounce.eml" s3.upload_email_from_bytesio( bounce_email_file_path, open(os.path.join(ROOT_DIR, "local_data", "email_tests", "2.eml"), "rb"), "download.eml", ) refused_email = RefusedEmail.create( path=bounce_email_file_path, full_report_path=bounce_email_file_path, user_id=user.id, commit=True, ) contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", commit=True, ) EmailLog.create( user_id=user.id, contact_id=contact.id, refused_email_id=refused_email.id, bounced=True, commit=True, ) LifetimeCoupon.create(code="coupon", nb_used=10, commit=True) # Create a subscription for user Subscription.create( user_id=user.id, cancel_url="https://checkout.paddle.com/subscription/cancel?user=1234", update_url="https://checkout.paddle.com/subscription/update?user=1234", subscription_id="123", event_time=arrow.now(), next_bill_date=arrow.now().shift(days=10).date(), plan=PlanEnum.monthly, commit=True, ) CoinbaseSubscription.create(user_id=user.id, end_at=arrow.now().shift(days=10), commit=True) api_key = ApiKey.create(user_id=user.id, name="Chrome") api_key.code = "code" api_key = ApiKey.create(user_id=user.id, name="Firefox") api_key.code = "codeFF" pgp_public_key = open(get_abs_path("local_data/public-pgp.asc")).read() m1 = Mailbox.create( user_id=user.id, email="*****@*****.**", verified=True, pgp_public_key=pgp_public_key, ) m1.pgp_finger_print = load_public_key(pgp_public_key) db.session.commit() for i in range(3): if i % 2 == 0: a = Alias.create(email=f"e{i}@{FIRST_ALIAS_DOMAIN}", user_id=user.id, mailbox_id=m1.id) else: a = Alias.create( email=f"e{i}@{FIRST_ALIAS_DOMAIN}", user_id=user.id, mailbox_id=user.default_mailbox_id, ) db.session.commit() if i % 5 == 0: if i % 2 == 0: AliasMailbox.create(alias_id=a.id, mailbox_id=user.default_mailbox_id) else: AliasMailbox.create(alias_id=a.id, mailbox_id=m1.id) db.session.commit() # some aliases don't have any activity # if i % 3 != 0: # contact = Contact.create( # user_id=user.id, # alias_id=a.id, # website_email=f"contact{i}@example.com", # reply_email=f"rep{i}@sl.local", # ) # db.session.commit() # for _ in range(3): # EmailLog.create(user_id=user.id, contact_id=contact.id) # db.session.commit() # have some disabled alias if i % 5 == 0: a.enabled = False db.session.commit() CustomDomain.create(user_id=user.id, domain="ab.cd", verified=True) CustomDomain.create(user_id=user.id, domain="very-long-domain.com.net.org", verified=True) db.session.commit() Directory.create(user_id=user.id, name="abcd") Directory.create(user_id=user.id, name="xyzt") db.session.commit() # Create a client client1 = Client.create_new(name="Demo", user_id=user.id) client1.oauth_client_id = "client-id" client1.oauth_client_secret = "client-secret" client1.published = True db.session.commit() RedirectUri.create(client_id=client1.id, uri="https://ab.com") client2 = Client.create_new(name="Demo 2", user_id=user.id) client2.oauth_client_id = "client-id2" client2.oauth_client_secret = "client-secret2" client2.published = True db.session.commit() ClientUser.create(user_id=user.id, client_id=client1.id, name="Fake Name") referral = Referral.create(user_id=user.id, code="REFCODE", name="First referral") db.session.commit() for i in range(6): Notification.create(user_id=user.id, message=f"""Hey hey <b>{i}</b> """ * 10) db.session.commit() User.create( email="*****@*****.**", password="******", activated=True, referral_id=referral.id, ) db.session.commit()
def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User): disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}" # Store the bounced email # generate a name for the email random_name = str(uuid.uuid4()) full_report_path = f"refused-emails/full-{random_name}.eml" s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name) file_path = None mailbox = None email_log: EmailLog = None orig_msg = get_orig_message_from_bounce(msg) if not orig_msg: # Some MTA does not return the original message in bounce message # nothing we can do here LOG.warning( "Cannot parse original message from bounce message %s %s %s %s", alias, user, contact, full_report_path, ) else: file_path = f"refused-emails/{random_name}.eml" s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()), random_name) try: mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER]) except TypeError: LOG.exception( "cannot parse mailbox from original message header %s", orig_msg[_MAILBOX_ID_HEADER], ) else: mailbox = Mailbox.get(mailbox_id) if not mailbox or mailbox.user_id != user.id: LOG.exception( "Tampered message mailbox_id %s, %s, %s, %s %s", mailbox_id, user, alias, contact, full_report_path, ) # cannot use this tampered mailbox, reset it mailbox = None # try to get the original email_log try: email_log_id = int(orig_msg[_EMAIL_LOG_ID_HEADER]) except TypeError: LOG.exception( "cannot parse email log from original message header %s", orig_msg[_EMAIL_LOG_ID_HEADER], ) else: email_log = EmailLog.get(email_log_id) refused_email = RefusedEmail.create(path=file_path, full_report_path=full_report_path, user_id=user.id) db.session.flush() LOG.d("Create refused email %s", refused_email) if not mailbox: LOG.debug("Try to get mailbox from bounce report") try: mailbox_id = int(get_header_from_bounce(msg, _MAILBOX_ID_HEADER)) except Exception: LOG.exception("cannot get mailbox-id from bounce report, %s", refused_email) else: mailbox = Mailbox.get(mailbox_id) if not mailbox or mailbox.user_id != user.id: LOG.exception( "Tampered message mailbox_id %s, %s, %s, %s %s", mailbox_id, user, alias, contact, full_report_path, ) mailbox = None if not email_log: LOG.d("Try to get email log from bounce report") try: email_log_id = int( get_header_from_bounce(msg, _EMAIL_LOG_ID_HEADER)) except Exception: LOG.exception("cannot get email log id from bounce report, %s", refused_email) else: email_log = EmailLog.get(email_log_id) # use the default mailbox as the last option if not mailbox: LOG.warning("Use %s default mailbox %s", alias, refused_email) mailbox = alias.mailbox # create a new email log as the last option if not email_log: LOG.warning("cannot get the original email_log, create a new one") email_log: EmailLog = EmailLog.create(contact_id=contact.id, user_id=contact.user_id) email_log.bounced = True email_log.refused_email_id = refused_email.id email_log.bounced_mailbox_id = mailbox.id db.session.commit() refused_email_url = (URL + f"/dashboard/refused_email?highlight_id=" + str(email_log.id)) nb_bounced = EmailLog.filter_by(contact_id=contact.id, bounced=True).count() if nb_bounced >= 2 and alias.cannot_be_disabled: LOG.warning("%s cannot be disabled", alias) # inform user if this is the first bounced email if nb_bounced == 1 or (nb_bounced >= 2 and alias.cannot_be_disabled): LOG.d( "Inform user %s about bounced email sent by %s to alias %s", user, contact.website_email, alias, ) send_email_with_rate_control( user, ALERT_BOUNCE_EMAIL, user.email, f"Email from {contact.website_email} to {alias.email} cannot be delivered to your inbox", render( "transactional/bounced-email.txt", name=user.name, alias=alias, website_email=contact.website_email, disable_alias_link=disable_alias_link, refused_email_url=refused_email_url, mailbox_email=mailbox.email, ), render( "transactional/bounced-email.html", name=user.name, alias=alias, website_email=contact.website_email, disable_alias_link=disable_alias_link, refused_email_url=refused_email_url, mailbox_email=mailbox.email, ), ) # disable the alias the second time email is bounced elif nb_bounced >= 2: LOG.d( "Bounce happens again with alias %s from %s. Disable alias now ", alias, contact.website_email, ) alias.enabled = False db.session.commit() send_email_with_rate_control( user, ALERT_BOUNCE_EMAIL, user.email, f"Alias {alias.email} has been disabled due to second undelivered email from {contact.website_email}", render( "transactional/automatic-disable-alias.txt", name=user.name, alias=alias, website_email=contact.website_email, refused_email_url=refused_email_url, mailbox_email=mailbox.email, ), render( "transactional/automatic-disable-alias.html", name=user.name, alias=alias, website_email=contact.website_email, refused_email_url=refused_email_url, mailbox_email=mailbox.email, ), )
def test_get_aliases_v2(flask_client): user = login(flask_client) a0 = Alias.create_new(user, "prefix0") a1 = Alias.create_new(user, "prefix1") Session.commit() # << Aliases have no activity >> r = flask_client.get("/api/v2/aliases?page_id=0") assert r.status_code == 200 r0 = r.json["aliases"][0] assert "name" in r0 # make sure a1 is returned before a0 assert r0["email"].startswith("prefix1") assert "id" in r0["mailbox"] assert "email" in r0["mailbox"] assert r0["mailboxes"] for mailbox in r0["mailboxes"]: assert "id" in mailbox assert "email" in mailbox assert "support_pgp" in r0 assert not r0["support_pgp"] assert "disable_pgp" in r0 assert not r0["disable_pgp"] # << Alias has some activities >> c0 = Contact.create( user_id=user.id, alias_id=a0.id, website_email="*****@*****.**", reply_email="re0@SL", commit=True, ) EmailLog.create(contact_id=c0.id, user_id=user.id, alias_id=c0.alias_id, commit=True) # a1 has more recent activity c1 = Contact.create( user_id=user.id, alias_id=a1.id, website_email="*****@*****.**", reply_email="re1@SL", commit=True, ) EmailLog.create(contact_id=c1.id, user_id=user.id, alias_id=c1.alias_id, commit=True) r = flask_client.get("/api/v2/aliases?page_id=0") assert r.status_code == 200 r0 = r.json["aliases"][0] assert r0["latest_activity"]["action"] == "forward" assert "timestamp" in r0["latest_activity"] assert r0["latest_activity"]["contact"]["email"] == "*****@*****.**" assert "name" in r0["latest_activity"]["contact"] assert "reverse_alias" in r0["latest_activity"]["contact"] assert "pinned" in r0
def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str): """ return whether an email has been delivered and the smtp status ("250 Message accepted", "550 Non-existent email address", etc) """ reply_email = rcpt_to.lower().strip() # reply_email must end with EMAIL_DOMAIN if not reply_email.endswith(EMAIL_DOMAIN): LOG.warning(f"Reply email {reply_email} has wrong domain") return False, "550 SL E2" contact = Contact.get_by(reply_email=reply_email) if not contact: LOG.warning(f"No such forward-email with {reply_email} as reply-email") return False, "550 SL E4 Email not exist" alias = contact.alias address: str = contact.alias.email alias_domain = address[address.find("@") + 1:] # alias must end with one of the ALIAS_DOMAINS or custom-domain if not email_belongs_to_alias_domains(alias.email): if not CustomDomain.get_by(domain=alias_domain): return False, "550 SL E5" user = alias.user mail_from = envelope.mail_from.lower().strip() # bounce email initiated by Postfix # can happen in case emails cannot be delivered to user-email # in this case Postfix will try to send a bounce report to original sender, which is # the "reply email" if mail_from == "<>": LOG.warning( "Bounce when sending to alias %s from %s, user %s", alias, contact, user, ) handle_bounce(contact, alias, msg, user) return False, "550 SL E6" mailbox = Mailbox.get_by(email=mail_from, user_id=user.id) if not mailbox or mailbox not in alias.mailboxes: # only mailbox can send email to the reply-email handle_unknown_mailbox(envelope, msg, reply_email, user, alias) return False, "550 SL E7" if ENFORCE_SPF and mailbox.force_spf: ip = msg[_IP_HEADER] if not spf_pass(ip, envelope, mailbox, user, alias, contact.website_email, msg): # cannot use 4** here as sender will retry. 5** because that generates bounce report return True, "250 SL E11" delete_header(msg, _IP_HEADER) delete_header(msg, "DKIM-Signature") delete_header(msg, "Received") # make the email comes from alias from_header = alias.email # add alias name from alias if alias.name: LOG.d("Put alias name in from header") from_header = formataddr((alias.name, alias.email)) elif alias.custom_domain: LOG.d("Put domain default alias name in from header") # add alias name from domain if alias.custom_domain.name: from_header = formataddr((alias.custom_domain.name, alias.email)) add_or_replace_header(msg, "From", from_header) # some email providers like ProtonMail adds automatically the Reply-To field # make sure to delete it delete_header(msg, "Reply-To") # remove sender header if present as this could reveal user real email delete_header(msg, "Sender") delete_header(msg, "X-Sender") replace_header_when_reply(msg, alias, "To") replace_header_when_reply(msg, alias, "Cc") # Received-SPF is injected by postfix-policyd-spf-python can reveal user original email delete_header(msg, "Received-SPF") LOG.d( "send email from %s to %s, mail_options:%s,rcpt_options:%s", alias.email, contact.website_email, envelope.mail_options, envelope.rcpt_options, ) # replace "*****@*****.**" by the contact email in the email body # as this is usually included when replying if user.replace_reverse_alias: if msg.is_multipart(): for part in msg.walk(): if part.get_content_maintype() != "text": continue part = replace_str_in_msg(part, reply_email, contact.website_email) else: msg = replace_str_in_msg(msg, reply_email, contact.website_email) if alias_domain in ALIAS_DOMAINS: add_dkim_signature(msg, alias_domain) # add DKIM-Signature for custom-domain alias else: custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain) if custom_domain.dkim_verified: add_dkim_signature(msg, alias_domain) # create PGP email if needed if contact.pgp_finger_print and user.is_premium(): LOG.d("Encrypt message for contact %s", contact) try: msg = prepare_pgp_message(msg, contact.pgp_finger_print) except PGPException: LOG.exception("Cannot encrypt message %s -> %s. %s %s", alias, contact, mailbox, user) # so the client can retry later return False, "421 SL E13 Retry later" try: smtp.sendmail( alias.email, contact.website_email, msg.as_bytes(), envelope.mail_options, envelope.rcpt_options, ) except Exception: LOG.exception("Cannot send email from %s to %s", alias, contact) send_email( mailbox.email, f"Email cannot be sent to {contact.email} from {alias.email}", render( "transactional/reply-error.txt", user=user, alias=alias, contact=contact, contact_domain=get_email_domain_part(contact.email), ), render( "transactional/reply-error.html", user=user, alias=alias, contact=contact, contact_domain=get_email_domain_part(contact.email), ), ) else: EmailLog.create(contact_id=contact.id, is_reply=True, user_id=contact.user_id) db.session.commit() return True, "250 Message accepted for delivery"
def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User): address = alias.email email_log: EmailLog = EmailLog.create(contact_id=contact.id, bounced=True, user_id=contact.user_id) db.session.commit() nb_bounced = EmailLog.filter_by(contact_id=contact.id, bounced=True).count() disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}" # <<< Store the bounced email >>> # generate a name for the email random_name = str(uuid.uuid4()) full_report_path = f"refused-emails/full-{random_name}.eml" s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name) orig_msg = get_orig_message_from_bounce(msg) if not orig_msg: LOG.error( "Cannot parse original message from bounce message %s %s %s %s", alias, user, contact, full_report_path, ) return file_path = f"refused-emails/{random_name}.eml" s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()), random_name) # <<< END Store the bounced email >>> mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER]) mailbox = Mailbox.get(mailbox_id) if not mailbox or mailbox.user_id != user.id: LOG.error( "Tampered message mailbox_id %s, %s, %s, %s %s", mailbox_id, user, alias, contact, full_report_path, ) return refused_email = RefusedEmail.create(path=file_path, full_report_path=full_report_path, user_id=user.id) db.session.flush() email_log.refused_email_id = refused_email.id email_log.bounced_mailbox_id = mailbox.id db.session.commit() LOG.d("Create refused email %s", refused_email) refused_email_url = (URL + f"/dashboard/refused_email?highlight_id=" + str(email_log.id)) # inform user if this is the first bounced email if nb_bounced == 1: LOG.d( "Inform user %s about bounced email sent by %s to alias %s", user, contact.website_email, address, ) send_email_with_rate_control( user, ALERT_BOUNCE_EMAIL, # use user mail here as only user is authenticated to see the refused email user.email, f"Email from {contact.website_email} to {address} cannot be delivered to your inbox", render( "transactional/bounced-email.txt", name=user.name, alias=alias, website_email=contact.website_email, disable_alias_link=disable_alias_link, refused_email_url=refused_email_url, mailbox_email=mailbox.email, ), render( "transactional/bounced-email.html", name=user.name, alias=alias, website_email=contact.website_email, disable_alias_link=disable_alias_link, refused_email_url=refused_email_url, mailbox_email=mailbox.email, ), # cannot include bounce email as it can contain spammy text # bounced_email=msg, ) # disable the alias the second time email is bounced elif nb_bounced >= 2: LOG.d( "Bounce happens again with alias %s from %s. Disable alias now ", address, contact.website_email, ) alias.enabled = False db.session.commit() send_email_with_rate_control( user, ALERT_BOUNCE_EMAIL, # use user mail here as only user is authenticated to see the refused email user.email, f"Alias {address} has been disabled due to second undelivered email from {contact.website_email}", render( "transactional/automatic-disable-alias.txt", name=user.name, alias=alias, website_email=contact.website_email, refused_email_url=refused_email_url, mailbox_email=mailbox.email, ), render( "transactional/automatic-disable-alias.html", name=user.name, alias=alias, website_email=contact.website_email, refused_email_url=refused_email_url, mailbox_email=mailbox.email, ), # cannot include bounce email as it can contain spammy text # bounced_email=msg, )
def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str): """ return whether an email has been delivered and the smtp status ("250 Message accepted", "550 Non-existent email address", etc) """ reply_email = rcpt_to.lower().strip() # reply_email must end with EMAIL_DOMAIN if not reply_email.endswith(EMAIL_DOMAIN): LOG.warning(f"Reply email {reply_email} has wrong domain") return False, "550 SL E2" contact = Contact.get_by(reply_email=reply_email) if not contact: LOG.warning(f"No such forward-email with {reply_email} as reply-email") return False, "550 SL E4" alias = contact.alias address: str = contact.alias.email alias_domain = address[address.find("@") + 1:] # alias must end with one of the ALIAS_DOMAINS or custom-domain if not email_belongs_to_alias_domains(alias.email): if not CustomDomain.get_by(domain=alias_domain): return False, "550 SL E5" user = alias.user mail_from = envelope.mail_from.lower().strip() # bounce email initiated by Postfix # can happen in case emails cannot be delivered to user-email # in this case Postfix will try to send a bounce report to original sender, which is # the "reply email" if mail_from == "<>": LOG.warning( "Bounce when sending to alias %s from %s, user %s", alias, contact, user, ) handle_bounce(contact, alias, msg, user) return False, "550 SL E6" mailbox = Mailbox.get_by(email=mail_from, user_id=user.id) if not mailbox or mailbox not in alias.mailboxes: # only mailbox can send email to the reply-email handle_unknown_mailbox(envelope, msg, reply_email, user, alias) return False, "550 SL E7" if ENFORCE_SPF and mailbox.force_spf: ip = msg[_IP_HEADER] if not spf_pass(ip, envelope, mailbox, user, alias, contact.website_email, msg): # cannot use 4** here as sender will retry. 5** because that generates bounce report return True, "250 SL E11" delete_header(msg, _IP_HEADER) delete_header(msg, "DKIM-Signature") delete_header(msg, "Received") # make the email comes from alias from_header = alias.email # add alias name from alias if alias.name: LOG.d("Put alias name in from header") from_header = formataddr((alias.name, alias.email)) elif alias.custom_domain: LOG.d("Put domain default alias name in from header") # add alias name from domain if alias.custom_domain.name: from_header = formataddr((alias.custom_domain.name, alias.email)) add_or_replace_header(msg, "From", from_header) # some email providers like ProtonMail adds automatically the Reply-To field # make sure to delete it delete_header(msg, "Reply-To") # remove sender header if present as this could reveal user real email delete_header(msg, "Sender") delete_header(msg, "X-Sender") replace_header_when_reply(msg, alias, "To") replace_header_when_reply(msg, alias, "Cc") # Received-SPF is injected by postfix-policyd-spf-python can reveal user original email delete_header(msg, "Received-SPF") LOG.d( "send email from %s to %s, mail_options:%s,rcpt_options:%s", alias.email, contact.website_email, envelope.mail_options, envelope.rcpt_options, ) if alias_domain in ALIAS_DOMAINS: add_dkim_signature(msg, alias_domain) # add DKIM-Signature for custom-domain alias else: custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain) if custom_domain.dkim_verified: add_dkim_signature(msg, alias_domain) smtp.sendmail( alias.email, contact.website_email, msg.as_bytes(), envelope.mail_options, envelope.rcpt_options, ) EmailLog.create(contact_id=contact.id, is_reply=True, user_id=contact.user_id) db.session.commit() return True, "250 Message accepted for delivery"
def test_get_aliases_v2(flask_client): user = User.create(email="[email protected]", password="******", name="Test User", activated=True) db.session.commit() # create api_key api_key = ApiKey.create(user.id, "for test") db.session.commit() a0 = Alias.create_new(user, "prefix0") a1 = Alias.create_new(user, "prefix1") db.session.commit() # add activity for a0 c0 = Contact.create( user_id=user.id, alias_id=a0.id, website_email="*****@*****.**", reply_email="re0@SL", ) db.session.commit() EmailLog.create(contact_id=c0.id, user_id=user.id) db.session.commit() # a1 has more recent activity c1 = Contact.create( user_id=user.id, alias_id=a1.id, website_email="*****@*****.**", reply_email="re1@SL", ) db.session.commit() EmailLog.create(contact_id=c1.id, user_id=user.id) db.session.commit() # get aliases v2 r = flask_client.get( url_for("api.get_aliases_v2", page_id=0), headers={"Authentication": api_key.code}, ) assert r.status_code == 200 # make sure a1 is returned before a0 r0 = r.json["aliases"][0] # r0 will have the following format # { # "creation_date": "2020-04-25 21:10:01+00:00", # "creation_timestamp": 1587849001, # "email": "*****@*****.**", # "enabled": true, # "id": 3, # "name": "Hey hey", # "latest_activity": { # "action": "forward", # "contact": { # "email": "*****@*****.**", # "name": null, # "reverse_alias": "\"c1 at example.com\" <re1@SL>" # }, # "timestamp": 1587849001 # }, # "mailbox": { # "email": "[email protected]", # "id": 1 # }, # "nb_block": 0, # "nb_forward": 1, # "nb_reply": 0, # "note": null # } assert "name" in r0 assert r0["email"].startswith("prefix1") assert r0["latest_activity"]["action"] == "forward" assert "timestamp" in r0["latest_activity"] assert r0["latest_activity"]["contact"]["email"] == "*****@*****.**" assert "name" in r0["latest_activity"]["contact"] assert "reverse_alias" in r0["latest_activity"]["contact"] assert "id" in r0["mailbox"] assert "email" in r0["mailbox"] assert r0["mailboxes"] for mailbox in r0["mailboxes"]: assert "id" in mailbox assert "email" in mailbox
def test_get_aliases_v2(flask_client): user = User.create(email="[email protected]", password="******", name="Test User", activated=True) db.session.commit() # create api_key api_key = ApiKey.create(user.id, "for test") db.session.commit() a0 = Alias.create_new(user, "prefix0") a1 = Alias.create_new(user, "prefix1") db.session.commit() # << Aliases have no activity >> r = flask_client.get( url_for("api.get_aliases_v2", page_id=0), headers={"Authentication": api_key.code}, ) assert r.status_code == 200 r0 = r.json["aliases"][0] assert "name" in r0 # make sure a1 is returned before a0 assert r0["email"].startswith("prefix1") assert "id" in r0["mailbox"] assert "email" in r0["mailbox"] assert r0["mailboxes"] for mailbox in r0["mailboxes"]: assert "id" in mailbox assert "email" in mailbox assert "support_pgp" in r0 assert not r0["support_pgp"] assert "disable_pgp" in r0 assert not r0["disable_pgp"] # << Alias has some activities >> c0 = Contact.create( user_id=user.id, alias_id=a0.id, website_email="*****@*****.**", reply_email="re0@SL", ) db.session.commit() EmailLog.create(contact_id=c0.id, user_id=user.id) db.session.commit() # a1 has more recent activity c1 = Contact.create( user_id=user.id, alias_id=a1.id, website_email="*****@*****.**", reply_email="re1@SL", ) db.session.commit() EmailLog.create(contact_id=c1.id, user_id=user.id) db.session.commit() # get aliases v2 r = flask_client.get( url_for("api.get_aliases_v2", page_id=0), headers={"Authentication": api_key.code}, ) assert r.status_code == 200 r0 = r.json["aliases"][0] assert r0["latest_activity"]["action"] == "forward" assert "timestamp" in r0["latest_activity"] assert r0["latest_activity"]["contact"]["email"] == "*****@*****.**" assert "name" in r0["latest_activity"]["contact"] assert "reverse_alias" in r0["latest_activity"]["contact"] assert "pinned" in r0
def fake_data(): LOG.d("create fake data") # Create a user user = User.create( email="*****@*****.**", name="John Wick", password="******", activated=True, is_admin=True, # enable_otp=True, otp_secret="base32secret3232", intro_shown=True, fido_uuid=None, ) user.trial_end = None Session.commit() # add a profile picture file_path = "profile_pic.svg" s3.upload_from_bytesio( file_path, open(os.path.join(ROOT_DIR, "static", "default-icon.svg"), "rb"), content_type="image/svg", ) file = File.create(user_id=user.id, path=file_path, commit=True) user.profile_picture_id = file.id Session.commit() # create a bounced email alias = Alias.create_new_random(user) Session.commit() bounce_email_file_path = "bounce.eml" s3.upload_email_from_bytesio( bounce_email_file_path, open(os.path.join(ROOT_DIR, "local_data", "email_tests", "2.eml"), "rb"), "download.eml", ) refused_email = RefusedEmail.create( path=bounce_email_file_path, full_report_path=bounce_email_file_path, user_id=user.id, commit=True, ) contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", commit=True, ) EmailLog.create( user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id, refused_email_id=refused_email.id, bounced=True, commit=True, ) LifetimeCoupon.create(code="lifetime-coupon", nb_used=10, commit=True) Coupon.create(code="coupon", commit=True) # Create a subscription for user Subscription.create( user_id=user.id, cancel_url="https://checkout.paddle.com/subscription/cancel?user=1234", update_url="https://checkout.paddle.com/subscription/update?user=1234", subscription_id="123", event_time=arrow.now(), next_bill_date=arrow.now().shift(days=10).date(), plan=PlanEnum.monthly, commit=True, ) CoinbaseSubscription.create(user_id=user.id, end_at=arrow.now().shift(days=10), commit=True) api_key = ApiKey.create(user_id=user.id, name="Chrome") api_key.code = "code" api_key = ApiKey.create(user_id=user.id, name="Firefox") api_key.code = "codeFF" pgp_public_key = open(get_abs_path("local_data/public-pgp.asc")).read() m1 = Mailbox.create( user_id=user.id, email="*****@*****.**", verified=True, pgp_public_key=pgp_public_key, ) m1.pgp_finger_print = load_public_key(pgp_public_key) Session.commit() # [email protected] is in a LOT of data breaches Alias.create(email="*****@*****.**", user_id=user.id, mailbox_id=m1.id) for i in range(3): if i % 2 == 0: a = Alias.create(email=f"e{i}@{FIRST_ALIAS_DOMAIN}", user_id=user.id, mailbox_id=m1.id) else: a = Alias.create( email=f"e{i}@{FIRST_ALIAS_DOMAIN}", user_id=user.id, mailbox_id=user.default_mailbox_id, ) Session.commit() if i % 5 == 0: if i % 2 == 0: AliasMailbox.create(alias_id=a.id, mailbox_id=user.default_mailbox_id) else: AliasMailbox.create(alias_id=a.id, mailbox_id=m1.id) Session.commit() # some aliases don't have any activity # if i % 3 != 0: # contact = Contact.create( # user_id=user.id, # alias_id=a.id, # website_email=f"contact{i}@example.com", # reply_email=f"rep{i}@sl.local", # ) # Session.commit() # for _ in range(3): # EmailLog.create(user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id) # Session.commit() # have some disabled alias if i % 5 == 0: a.enabled = False Session.commit() custom_domain1 = CustomDomain.create(user_id=user.id, domain="ab.cd", verified=True) Session.commit() Alias.create( user_id=user.id, email="*****@*****.**", mailbox_id=user.default_mailbox_id, custom_domain_id=custom_domain1.id, commit=True, ) Alias.create( user_id=user.id, email="*****@*****.**", mailbox_id=user.default_mailbox_id, custom_domain_id=custom_domain1.id, commit=True, ) Directory.create(user_id=user.id, name="abcd") Directory.create(user_id=user.id, name="xyzt") Session.commit() # Create a client client1 = Client.create_new(name="Demo", user_id=user.id) client1.oauth_client_id = "client-id" client1.oauth_client_secret = "client-secret" Session.commit() RedirectUri.create(client_id=client1.id, uri="https://your-website.com/oauth-callback") client2 = Client.create_new(name="Demo 2", user_id=user.id) client2.oauth_client_id = "client-id2" client2.oauth_client_secret = "client-secret2" Session.commit() ClientUser.create(user_id=user.id, client_id=client1.id, name="Fake Name") referral = Referral.create(user_id=user.id, code="Website", name="First referral") Referral.create(user_id=user.id, code="Podcast", name="First referral") Payout.create(user_id=user.id, amount=1000, number_upgraded_account=100, payment_method="BTC") Payout.create( user_id=user.id, amount=5000, number_upgraded_account=200, payment_method="PayPal", ) Session.commit() for i in range(6): Notification.create(user_id=user.id, message=f"""Hey hey <b>{i}</b> """ * 10) Session.commit() user2 = User.create( email="*****@*****.**", password="******", activated=True, referral_id=referral.id, ) Mailbox.create(user_id=user2.id, email="*****@*****.**", verified=True) Session.commit() ManualSubscription.create( user_id=user2.id, end_at=arrow.now().shift(years=1, days=1), comment="Local manual", commit=True, ) SLDomain.create(domain="premium.com", premium_only=True, commit=True) hibp1 = Hibp.create(name="first breach", description="breach description", commit=True) hibp2 = Hibp.create(name="second breach", description="breach description", commit=True) breached_alias1 = Alias.create(email="*****@*****.**", user_id=user.id, mailbox_id=m1.id, commit=True) breached_alias2 = Alias.create(email="*****@*****.**", user_id=user.id, mailbox_id=m1.id, commit=True) AliasHibp.create(hibp_id=hibp1.id, alias_id=breached_alias1.id) AliasHibp.create(hibp_id=hibp2.id, alias_id=breached_alias2.id) # old domain will have ownership_verified=True CustomDomain.create(user_id=user.id, domain="old.com", verified=True, ownership_verified=True)