def test_rate_limited_forward_phase_for_alias(flask_client):
user = User.create(email="[email protected]",
password="******",
name="Test User",
activated=True)
db.session.commit()
# no rate limiting for a new alias
alias = Alias.create_new_random(user)
db.session.commit()
assert not rate_limited_for_alias(alias)
# rate limit when there's a previous activity on alias
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
)
db.session.commit()
for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_ALIAS + 1):
EmailLog.create(user_id=user.id, contact_id=contact.id)
db.session.commit()
assert rate_limited_for_alias(alias)
def test_rate_limited_reply_phase(flask_client):
# no rate limiting when reply_email does not exist
assert not rate_limited_reply_phase("*****@*****.**")
user = User.create(email="[email protected]",
password="******",
name="Test User",
activated=True)
db.session.commit()
alias = Alias.create_new_random(user)
db.session.commit()
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
)
db.session.commit()
for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_ALIAS + 1):
EmailLog.create(user_id=user.id, contact_id=contact.id)
db.session.commit()
assert rate_limited_reply_phase("*****@*****.**")
def test_should_disable_bounces_account(flask_client):
"""if an account has more than 10 bounces every day for at least 5 days in the last 10 days, disable alias"""
user = login(flask_client)
alias = Alias.create_new_random(user)
Session.commit()
# create a lot of bounces on alias
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
for day in range(6):
for _ in range(10):
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
alias_id=contact.alias_id,
commit=True,
bounced=True,
created_at=arrow.now().shift(days=-day),
)
alias2 = Alias.create_new_random(user)
assert should_disable(alias2)
def test_should_disable_bounces_every_day(flask_client):
"""if an alias has bounces every day at least 9 days in the last 10 days, disable alias"""
user = login(flask_client)
alias = Alias.create_new_random(user)
Session.commit()
assert not should_disable(alias)
# create a lot of bounce on this alias
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
for i in range(9):
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
alias_id=contact.alias_id,
commit=True,
bounced=True,
created_at=arrow.now().shift(days=-i),
)
assert should_disable(alias)
def test_alias_activities(flask_client):
user = User.create(email="[email protected]",
password="******",
name="Test User",
activated=True)
Session.commit()
# create api_key
api_key = ApiKey.create(user.id, "for test")
Session.commit()
alias = Alias.create_new_random(user)
Session.commit()
# create some alias log
contact = Contact.create(
website_email="*****@*****.**",
reply_email="[email protected]",
alias_id=alias.id,
user_id=alias.user_id,
)
Session.commit()
for _ in range(int(PAGE_LIMIT / 2)):
EmailLog.create(
contact_id=contact.id,
is_reply=True,
user_id=contact.user_id,
alias_id=contact.alias_id,
)
for _ in range(int(PAGE_LIMIT / 2) + 2):
EmailLog.create(
contact_id=contact.id,
blocked=True,
user_id=contact.user_id,
alias_id=contact.alias_id,
)
r = flask_client.get(
url_for("api.get_alias_activities", alias_id=alias.id, page_id=0),
headers={"Authentication": api_key.code},
)
assert r.status_code == 200
assert len(r.json["activities"]) == PAGE_LIMIT
for ac in r.json["activities"]:
assert ac["from"]
assert ac["to"]
assert ac["timestamp"]
assert ac["action"]
assert ac["reverse_alias"]
assert ac["reverse_alias_address"]
# second page, should return 1 or 2 results only
r = flask_client.get(
url_for("api.get_alias_activities", alias_id=alias.id, page_id=1),
headers={"Authentication": api_key.code},
)
assert len(r.json["activities"]) < 3
def test_should_disable_bounce_consecutive_days(flask_client):
user = login(flask_client)
alias = Alias.create_new_random(user)
Session.commit()
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
# create 6 bounce on this alias in the last 24h: alias is not disabled
for _ in range(6):
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
alias_id=contact.alias_id,
commit=True,
bounced=True,
)
assert not should_disable(alias)
# create 2 bounces in the last 7 days: alias should be disabled
for _ in range(2):
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
alias_id=contact.alias_id,
commit=True,
bounced=True,
created_at=arrow.now().shift(days=-3),
)
assert should_disable(alias)
def test_should_disable(flask_client):
user = User.create(
email="[email protected]",
password="******",
name="Test User",
activated=True,
include_sender_in_reverse_alias=True,
)
alias = Alias.create_new_random(user)
db.session.commit()
assert not should_disable(alias)
# create a lot of bounce on this alias
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
for _ in range(20):
EmailLog.create(user_id=user.id,
contact_id=contact.id,
commit=True,
bounced=True)
assert should_disable(alias)
# should not affect another alias
alias2 = Alias.create_new_random(user)
db.session.commit()
assert not should_disable(alias2)
def test_greylisting_needed_forward_phase_for_mailbox(flask_client):
user = User.create(
email="[email protected]", password="******", name="Test User", activated=True
)
db.session.commit()
alias = Alias.create_new_random(user)
db.session.commit()
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
)
db.session.commit()
for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_MAILBOX + 1):
EmailLog.create(user_id=user.id, contact_id=contact.id)
db.session.commit()
EmailLog.create(user_id=user.id, contact_id=contact.id)
# Create another alias with the same mailbox
# will be greylisted as there's a previous activity on mailbox
alias2 = Alias.create_new_random(user)
db.session.commit()
assert greylisting_needed_for_mailbox(alias2)
def test_alias_contacts(flask_client):
user = User.create(
email="[email protected]", password="******", name="Test User", activated=True
)
db.session.commit()
# create api_key
api_key = ApiKey.create(user.id, "for test")
db.session.commit()
alias = Alias.create_new_random(user)
db.session.commit()
# create some alias log
for i in range(PAGE_LIMIT + 1):
contact = Contact.create(
website_email=f"marketing-{i}@example.com",
reply_email=f"reply-{i}@a.b",
alias_id=alias.id,
user_id=alias.user_id,
)
db.session.commit()
EmailLog.create(
contact_id=contact.id,
is_reply=True,
user_id=contact.user_id,
alias_id=contact.alias_id,
)
db.session.commit()
r = flask_client.get(
url_for("api.get_alias_contacts_route", alias_id=alias.id, page_id=0),
headers={"Authentication": api_key.code},
)
assert r.status_code == 200
assert len(r.json["contacts"]) == PAGE_LIMIT
for ac in r.json["contacts"]:
assert ac["creation_date"]
assert ac["creation_timestamp"]
assert ac["last_email_sent_date"]
assert ac["last_email_sent_timestamp"]
assert ac["contact"]
assert ac["reverse_alias"]
assert ac["reverse_alias_address"]
# second page, should return 1 result only
r = flask_client.get(
url_for("api.get_alias_contacts_route", alias_id=alias.id, page_id=1),
headers={"Authentication": api_key.code},
)
assert len(r.json["contacts"]) == 1
async def handle_forward(envelope, smtp: SMTP, msg: Message,
rcpt_to: str) -> List[Tuple[bool, str]]:
"""return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
address = rcpt_to.lower().strip() # alias@SL
alias = Alias.get_by(email=address)
if not alias:
LOG.d("alias %s not exist. Try to see if it can be created on the fly",
address)
alias = try_auto_create(address)
if not alias:
LOG.d("alias %s cannot be created on-the-fly, return 550", address)
return [(False, "550 SL E3 Email not exist")]
mail_from = envelope.mail_from.lower().strip()
for mb in alias.mailboxes:
# email send from a mailbox to alias
if mb.email.lower().strip() == mail_from:
LOG.exception("cycle email sent from %s to %s", mb, alias)
handle_email_sent_to_ourself(alias, mb, msg, alias.user)
return [(True, "250 Message accepted for delivery")]
contact = get_or_create_contact(msg["From"], envelope.mail_from, alias)
email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id)
db.session.commit()
if not alias.enabled:
LOG.d("%s is disabled, do not forward", alias)
email_log.blocked = True
db.session.commit()
# do not return 5** to allow user to receive emails later when alias is enabled
return [(True, "250 Message accepted for delivery")]
user = alias.user
ret = []
mailboxes = alias.mailboxes
# no need to create a copy of message
if len(mailboxes) == 1:
mailbox = mailboxes[0]
ret.append(await
forward_email_to_mailbox(alias, msg, email_log, contact,
envelope, smtp, mailbox, user))
# create a copy of message for each forward
else:
for mailbox in mailboxes:
ret.append(await
forward_email_to_mailbox(alias, copy(msg), email_log,
contact, envelope, smtp,
mailbox, user))
return ret
def test_alias_contacts(flask_client):
user = login(flask_client)
alias = Alias.create_new_random(user)
Session.commit()
# create some alias log
for i in range(PAGE_LIMIT + 1):
contact = Contact.create(
website_email=f"marketing-{i}@example.com",
reply_email=f"reply-{i}@a.b",
alias_id=alias.id,
user_id=alias.user_id,
)
Session.commit()
EmailLog.create(
contact_id=contact.id,
is_reply=True,
user_id=contact.user_id,
alias_id=contact.alias_id,
)
Session.commit()
r = flask_client.get(f"/api/aliases/{alias.id}/contacts?page_id=0")
assert r.status_code == 200
assert len(r.json["contacts"]) == PAGE_LIMIT
for ac in r.json["contacts"]:
assert ac["creation_date"]
assert ac["creation_timestamp"]
assert ac["last_email_sent_date"]
assert ac["last_email_sent_timestamp"]
assert ac["contact"]
assert ac["reverse_alias"]
assert ac["reverse_alias_address"]
assert "block_forward" in ac
# second page, should return 1 result only
r = flask_client.get(f"/api/aliases/{alias.id}/contacts?page_id=1")
assert len(r.json["contacts"]) == 1
def post(self):
data = request.get_json()
now = datetime.datetime.now()
if data.get('eta') is not None:
send_at = datetime.datetime.strptime(data['eta'],
'%Y-%m-%d %H:%M:%S')
if send_at < now:
raise ScheduleTimeException(
'schedule time:{} error'.format(send_at))
else:
countdown = data['countdown']
send_at = datetime.datetime.now() + datetime.timedelta(**countdown)
email = EmailLog.create(subject=data['subject'],
body=data['body'],
to=data['to'],
send_at=send_at)
if send_at - now <= datetime.timedelta(minutes=15):
send_email.apply_async(args=(email.id, ), eta=send_at)
return jsonify()
def handle_forward(envelope, smtp: SMTP, msg: Message,
rcpt_to: str) -> List[Tuple[bool, str]]:
"""return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
address = rcpt_to.lower().strip() # alias@SL
alias = Alias.get_by(email=address)
if not alias:
LOG.d("alias %s not exist. Try to see if it can be created on the fly",
address)
alias = try_auto_create(address)
if not alias:
LOG.d("alias %s cannot be created on-the-fly, return 550", address)
return [(False, "550 SL E3")]
contact = get_or_create_contact(msg["From"], envelope.mail_from, alias)
email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id)
if not alias.enabled:
LOG.d("%s is disabled, do not forward", alias)
email_log.blocked = True
db.session.commit()
# do not return 5** to allow user to receive emails later when alias is enabled
return [(True, "250 Message accepted for delivery")]
user = alias.user
ret = []
for mailbox in alias.mailboxes:
ret.append(
forward_email_to_mailbox(alias, msg, email_log, contact, envelope,
smtp, mailbox, user))
return ret
def fake_data():
LOG.d("create fake data")
# Remove db if exist
if os.path.exists("db.sqlite"):
LOG.d("remove existing db file")
os.remove("db.sqlite")
# Create all tables
db.create_all()
# Create a user
user = User.create(
email="*****@*****.**",
name="John Wick",
password="******",
activated=True,
is_admin=True,
enable_otp=False,
otp_secret="base32secret3232",
intro_shown=True,
fido_uuid=None,
)
user.include_sender_in_reverse_alias = None
user.trial_end = None
db.session.commit()
# add a profile picture
file_path = "profile_pic.svg"
s3.upload_from_bytesio(
file_path,
open(os.path.join(ROOT_DIR, "static", "default-icon.svg"), "rb"),
content_type="image/svg",
)
file = File.create(user_id=user.id, path=file_path, commit=True)
user.profile_picture_id = file.id
db.session.commit()
# create a bounced email
alias = Alias.create_new_random(user)
db.session.commit()
bounce_email_file_path = "bounce.eml"
s3.upload_email_from_bytesio(
bounce_email_file_path,
open(os.path.join(ROOT_DIR, "local_data", "email_tests", "2.eml"),
"rb"),
"download.eml",
)
refused_email = RefusedEmail.create(
path=bounce_email_file_path,
full_report_path=bounce_email_file_path,
user_id=user.id,
commit=True,
)
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
refused_email_id=refused_email.id,
bounced=True,
commit=True,
)
LifetimeCoupon.create(code="coupon", nb_used=10, commit=True)
# Create a subscription for user
Subscription.create(
user_id=user.id,
cancel_url="https://checkout.paddle.com/subscription/cancel?user=1234",
update_url="https://checkout.paddle.com/subscription/update?user=1234",
subscription_id="123",
event_time=arrow.now(),
next_bill_date=arrow.now().shift(days=10).date(),
plan=PlanEnum.monthly,
commit=True,
)
CoinbaseSubscription.create(user_id=user.id,
end_at=arrow.now().shift(days=10),
commit=True)
api_key = ApiKey.create(user_id=user.id, name="Chrome")
api_key.code = "code"
api_key = ApiKey.create(user_id=user.id, name="Firefox")
api_key.code = "codeFF"
pgp_public_key = open(get_abs_path("local_data/public-pgp.asc")).read()
m1 = Mailbox.create(
user_id=user.id,
email="*****@*****.**",
verified=True,
pgp_public_key=pgp_public_key,
)
m1.pgp_finger_print = load_public_key(pgp_public_key)
db.session.commit()
for i in range(3):
if i % 2 == 0:
a = Alias.create(email=f"e{i}@{FIRST_ALIAS_DOMAIN}",
user_id=user.id,
mailbox_id=m1.id)
else:
a = Alias.create(
email=f"e{i}@{FIRST_ALIAS_DOMAIN}",
user_id=user.id,
mailbox_id=user.default_mailbox_id,
)
db.session.commit()
if i % 5 == 0:
if i % 2 == 0:
AliasMailbox.create(alias_id=a.id,
mailbox_id=user.default_mailbox_id)
else:
AliasMailbox.create(alias_id=a.id, mailbox_id=m1.id)
db.session.commit()
# some aliases don't have any activity
# if i % 3 != 0:
# contact = Contact.create(
# user_id=user.id,
# alias_id=a.id,
# website_email=f"contact{i}@example.com",
# reply_email=f"rep{i}@sl.local",
# )
# db.session.commit()
# for _ in range(3):
# EmailLog.create(user_id=user.id, contact_id=contact.id)
# db.session.commit()
# have some disabled alias
if i % 5 == 0:
a.enabled = False
db.session.commit()
CustomDomain.create(user_id=user.id, domain="ab.cd", verified=True)
CustomDomain.create(user_id=user.id,
domain="very-long-domain.com.net.org",
verified=True)
db.session.commit()
Directory.create(user_id=user.id, name="abcd")
Directory.create(user_id=user.id, name="xyzt")
db.session.commit()
# Create a client
client1 = Client.create_new(name="Demo", user_id=user.id)
client1.oauth_client_id = "client-id"
client1.oauth_client_secret = "client-secret"
client1.published = True
db.session.commit()
RedirectUri.create(client_id=client1.id, uri="https://ab.com")
client2 = Client.create_new(name="Demo 2", user_id=user.id)
client2.oauth_client_id = "client-id2"
client2.oauth_client_secret = "client-secret2"
client2.published = True
db.session.commit()
ClientUser.create(user_id=user.id, client_id=client1.id, name="Fake Name")
referral = Referral.create(user_id=user.id,
code="REFCODE",
name="First referral")
db.session.commit()
for i in range(6):
Notification.create(user_id=user.id,
message=f"""Hey hey <b>{i}</b> """ * 10)
db.session.commit()
User.create(
email="*****@*****.**",
password="******",
activated=True,
referral_id=referral.id,
)
db.session.commit()
def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User):
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
# Store the bounced email
# generate a name for the email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/full-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()),
random_name)
file_path = None
mailbox = None
email_log: EmailLog = None
orig_msg = get_orig_message_from_bounce(msg)
if not orig_msg:
# Some MTA does not return the original message in bounce message
# nothing we can do here
LOG.warning(
"Cannot parse original message from bounce message %s %s %s %s",
alias,
user,
contact,
full_report_path,
)
else:
file_path = f"refused-emails/{random_name}.eml"
s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()),
random_name)
try:
mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER])
except TypeError:
LOG.exception(
"cannot parse mailbox from original message header %s",
orig_msg[_MAILBOX_ID_HEADER],
)
else:
mailbox = Mailbox.get(mailbox_id)
if not mailbox or mailbox.user_id != user.id:
LOG.exception(
"Tampered message mailbox_id %s, %s, %s, %s %s",
mailbox_id,
user,
alias,
contact,
full_report_path,
)
# cannot use this tampered mailbox, reset it
mailbox = None
# try to get the original email_log
try:
email_log_id = int(orig_msg[_EMAIL_LOG_ID_HEADER])
except TypeError:
LOG.exception(
"cannot parse email log from original message header %s",
orig_msg[_EMAIL_LOG_ID_HEADER],
)
else:
email_log = EmailLog.get(email_log_id)
refused_email = RefusedEmail.create(path=file_path,
full_report_path=full_report_path,
user_id=user.id)
db.session.flush()
LOG.d("Create refused email %s", refused_email)
if not mailbox:
LOG.debug("Try to get mailbox from bounce report")
try:
mailbox_id = int(get_header_from_bounce(msg, _MAILBOX_ID_HEADER))
except Exception:
LOG.exception("cannot get mailbox-id from bounce report, %s",
refused_email)
else:
mailbox = Mailbox.get(mailbox_id)
if not mailbox or mailbox.user_id != user.id:
LOG.exception(
"Tampered message mailbox_id %s, %s, %s, %s %s",
mailbox_id,
user,
alias,
contact,
full_report_path,
)
mailbox = None
if not email_log:
LOG.d("Try to get email log from bounce report")
try:
email_log_id = int(
get_header_from_bounce(msg, _EMAIL_LOG_ID_HEADER))
except Exception:
LOG.exception("cannot get email log id from bounce report, %s",
refused_email)
else:
email_log = EmailLog.get(email_log_id)
# use the default mailbox as the last option
if not mailbox:
LOG.warning("Use %s default mailbox %s", alias, refused_email)
mailbox = alias.mailbox
# create a new email log as the last option
if not email_log:
LOG.warning("cannot get the original email_log, create a new one")
email_log: EmailLog = EmailLog.create(contact_id=contact.id,
user_id=contact.user_id)
email_log.bounced = True
email_log.refused_email_id = refused_email.id
email_log.bounced_mailbox_id = mailbox.id
db.session.commit()
refused_email_url = (URL + f"/dashboard/refused_email?highlight_id=" +
str(email_log.id))
nb_bounced = EmailLog.filter_by(contact_id=contact.id,
bounced=True).count()
if nb_bounced >= 2 and alias.cannot_be_disabled:
LOG.warning("%s cannot be disabled", alias)
# inform user if this is the first bounced email
if nb_bounced == 1 or (nb_bounced >= 2 and alias.cannot_be_disabled):
LOG.d(
"Inform user %s about bounced email sent by %s to alias %s",
user,
contact.website_email,
alias,
)
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
user.email,
f"Email from {contact.website_email} to {alias.email} cannot be delivered to your inbox",
render(
"transactional/bounced-email.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/bounced-email.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
)
# disable the alias the second time email is bounced
elif nb_bounced >= 2:
LOG.d(
"Bounce happens again with alias %s from %s. Disable alias now ",
alias,
contact.website_email,
)
alias.enabled = False
db.session.commit()
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
user.email,
f"Alias {alias.email} has been disabled due to second undelivered email from {contact.website_email}",
render(
"transactional/automatic-disable-alias.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/automatic-disable-alias.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
)
def test_get_aliases_v2(flask_client):
user = login(flask_client)
a0 = Alias.create_new(user, "prefix0")
a1 = Alias.create_new(user, "prefix1")
Session.commit()
# << Aliases have no activity >>
r = flask_client.get("/api/v2/aliases?page_id=0")
assert r.status_code == 200
r0 = r.json["aliases"][0]
assert "name" in r0
# make sure a1 is returned before a0
assert r0["email"].startswith("prefix1")
assert "id" in r0["mailbox"]
assert "email" in r0["mailbox"]
assert r0["mailboxes"]
for mailbox in r0["mailboxes"]:
assert "id" in mailbox
assert "email" in mailbox
assert "support_pgp" in r0
assert not r0["support_pgp"]
assert "disable_pgp" in r0
assert not r0["disable_pgp"]
# << Alias has some activities >>
c0 = Contact.create(
user_id=user.id,
alias_id=a0.id,
website_email="*****@*****.**",
reply_email="re0@SL",
commit=True,
)
EmailLog.create(contact_id=c0.id,
user_id=user.id,
alias_id=c0.alias_id,
commit=True)
# a1 has more recent activity
c1 = Contact.create(
user_id=user.id,
alias_id=a1.id,
website_email="*****@*****.**",
reply_email="re1@SL",
commit=True,
)
EmailLog.create(contact_id=c1.id,
user_id=user.id,
alias_id=c1.alias_id,
commit=True)
r = flask_client.get("/api/v2/aliases?page_id=0")
assert r.status_code == 200
r0 = r.json["aliases"][0]
assert r0["latest_activity"]["action"] == "forward"
assert "timestamp" in r0["latest_activity"]
assert r0["latest_activity"]["contact"]["email"] == "*****@*****.**"
assert "name" in r0["latest_activity"]["contact"]
assert "reverse_alias" in r0["latest_activity"]["contact"]
assert "pinned" in r0
def handle_reply(envelope, smtp: SMTP, msg: Message,
rcpt_to: str) -> (bool, str):
"""
return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
reply_email = rcpt_to.lower().strip()
# reply_email must end with EMAIL_DOMAIN
if not reply_email.endswith(EMAIL_DOMAIN):
LOG.warning(f"Reply email {reply_email} has wrong domain")
return False, "550 SL E2"
contact = Contact.get_by(reply_email=reply_email)
if not contact:
LOG.warning(f"No such forward-email with {reply_email} as reply-email")
return False, "550 SL E4 Email not exist"
alias = contact.alias
address: str = contact.alias.email
alias_domain = address[address.find("@") + 1:]
# alias must end with one of the ALIAS_DOMAINS or custom-domain
if not email_belongs_to_alias_domains(alias.email):
if not CustomDomain.get_by(domain=alias_domain):
return False, "550 SL E5"
user = alias.user
mail_from = envelope.mail_from.lower().strip()
# bounce email initiated by Postfix
# can happen in case emails cannot be delivered to user-email
# in this case Postfix will try to send a bounce report to original sender, which is
# the "reply email"
if mail_from == "<>":
LOG.warning(
"Bounce when sending to alias %s from %s, user %s",
alias,
contact,
user,
)
handle_bounce(contact, alias, msg, user)
return False, "550 SL E6"
mailbox = Mailbox.get_by(email=mail_from, user_id=user.id)
if not mailbox or mailbox not in alias.mailboxes:
# only mailbox can send email to the reply-email
handle_unknown_mailbox(envelope, msg, reply_email, user, alias)
return False, "550 SL E7"
if ENFORCE_SPF and mailbox.force_spf:
ip = msg[_IP_HEADER]
if not spf_pass(ip, envelope, mailbox, user, alias,
contact.website_email, msg):
# cannot use 4** here as sender will retry. 5** because that generates bounce report
return True, "250 SL E11"
delete_header(msg, _IP_HEADER)
delete_header(msg, "DKIM-Signature")
delete_header(msg, "Received")
# make the email comes from alias
from_header = alias.email
# add alias name from alias
if alias.name:
LOG.d("Put alias name in from header")
from_header = formataddr((alias.name, alias.email))
elif alias.custom_domain:
LOG.d("Put domain default alias name in from header")
# add alias name from domain
if alias.custom_domain.name:
from_header = formataddr((alias.custom_domain.name, alias.email))
add_or_replace_header(msg, "From", from_header)
# some email providers like ProtonMail adds automatically the Reply-To field
# make sure to delete it
delete_header(msg, "Reply-To")
# remove sender header if present as this could reveal user real email
delete_header(msg, "Sender")
delete_header(msg, "X-Sender")
replace_header_when_reply(msg, alias, "To")
replace_header_when_reply(msg, alias, "Cc")
# Received-SPF is injected by postfix-policyd-spf-python can reveal user original email
delete_header(msg, "Received-SPF")
LOG.d(
"send email from %s to %s, mail_options:%s,rcpt_options:%s",
alias.email,
contact.website_email,
envelope.mail_options,
envelope.rcpt_options,
)
# replace "*****@*****.**" by the contact email in the email body
# as this is usually included when replying
if user.replace_reverse_alias:
if msg.is_multipart():
for part in msg.walk():
if part.get_content_maintype() != "text":
continue
part = replace_str_in_msg(part, reply_email,
contact.website_email)
else:
msg = replace_str_in_msg(msg, reply_email, contact.website_email)
if alias_domain in ALIAS_DOMAINS:
add_dkim_signature(msg, alias_domain)
# add DKIM-Signature for custom-domain alias
else:
custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain)
if custom_domain.dkim_verified:
add_dkim_signature(msg, alias_domain)
# create PGP email if needed
if contact.pgp_finger_print and user.is_premium():
LOG.d("Encrypt message for contact %s", contact)
try:
msg = prepare_pgp_message(msg, contact.pgp_finger_print)
except PGPException:
LOG.exception("Cannot encrypt message %s -> %s. %s %s", alias,
contact, mailbox, user)
# so the client can retry later
return False, "421 SL E13 Retry later"
try:
smtp.sendmail(
alias.email,
contact.website_email,
msg.as_bytes(),
envelope.mail_options,
envelope.rcpt_options,
)
except Exception:
LOG.exception("Cannot send email from %s to %s", alias, contact)
send_email(
mailbox.email,
f"Email cannot be sent to {contact.email} from {alias.email}",
render(
"transactional/reply-error.txt",
user=user,
alias=alias,
contact=contact,
contact_domain=get_email_domain_part(contact.email),
),
render(
"transactional/reply-error.html",
user=user,
alias=alias,
contact=contact,
contact_domain=get_email_domain_part(contact.email),
),
)
else:
EmailLog.create(contact_id=contact.id,
is_reply=True,
user_id=contact.user_id)
db.session.commit()
return True, "250 Message accepted for delivery"
def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User):
address = alias.email
email_log: EmailLog = EmailLog.create(contact_id=contact.id,
bounced=True,
user_id=contact.user_id)
db.session.commit()
nb_bounced = EmailLog.filter_by(contact_id=contact.id,
bounced=True).count()
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
# <<< Store the bounced email >>>
# generate a name for the email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/full-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()),
random_name)
orig_msg = get_orig_message_from_bounce(msg)
if not orig_msg:
LOG.error(
"Cannot parse original message from bounce message %s %s %s %s",
alias,
user,
contact,
full_report_path,
)
return
file_path = f"refused-emails/{random_name}.eml"
s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()),
random_name)
# <<< END Store the bounced email >>>
mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER])
mailbox = Mailbox.get(mailbox_id)
if not mailbox or mailbox.user_id != user.id:
LOG.error(
"Tampered message mailbox_id %s, %s, %s, %s %s",
mailbox_id,
user,
alias,
contact,
full_report_path,
)
return
refused_email = RefusedEmail.create(path=file_path,
full_report_path=full_report_path,
user_id=user.id)
db.session.flush()
email_log.refused_email_id = refused_email.id
email_log.bounced_mailbox_id = mailbox.id
db.session.commit()
LOG.d("Create refused email %s", refused_email)
refused_email_url = (URL + f"/dashboard/refused_email?highlight_id=" +
str(email_log.id))
# inform user if this is the first bounced email
if nb_bounced == 1:
LOG.d(
"Inform user %s about bounced email sent by %s to alias %s",
user,
contact.website_email,
address,
)
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
# use user mail here as only user is authenticated to see the refused email
user.email,
f"Email from {contact.website_email} to {address} cannot be delivered to your inbox",
render(
"transactional/bounced-email.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/bounced-email.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
# cannot include bounce email as it can contain spammy text
# bounced_email=msg,
)
# disable the alias the second time email is bounced
elif nb_bounced >= 2:
LOG.d(
"Bounce happens again with alias %s from %s. Disable alias now ",
address,
contact.website_email,
)
alias.enabled = False
db.session.commit()
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
# use user mail here as only user is authenticated to see the refused email
user.email,
f"Alias {address} has been disabled due to second undelivered email from {contact.website_email}",
render(
"transactional/automatic-disable-alias.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/automatic-disable-alias.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
# cannot include bounce email as it can contain spammy text
# bounced_email=msg,
)
def handle_reply(envelope, smtp: SMTP, msg: Message,
rcpt_to: str) -> (bool, str):
"""
return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
reply_email = rcpt_to.lower().strip()
# reply_email must end with EMAIL_DOMAIN
if not reply_email.endswith(EMAIL_DOMAIN):
LOG.warning(f"Reply email {reply_email} has wrong domain")
return False, "550 SL E2"
contact = Contact.get_by(reply_email=reply_email)
if not contact:
LOG.warning(f"No such forward-email with {reply_email} as reply-email")
return False, "550 SL E4"
alias = contact.alias
address: str = contact.alias.email
alias_domain = address[address.find("@") + 1:]
# alias must end with one of the ALIAS_DOMAINS or custom-domain
if not email_belongs_to_alias_domains(alias.email):
if not CustomDomain.get_by(domain=alias_domain):
return False, "550 SL E5"
user = alias.user
mail_from = envelope.mail_from.lower().strip()
# bounce email initiated by Postfix
# can happen in case emails cannot be delivered to user-email
# in this case Postfix will try to send a bounce report to original sender, which is
# the "reply email"
if mail_from == "<>":
LOG.warning(
"Bounce when sending to alias %s from %s, user %s",
alias,
contact,
user,
)
handle_bounce(contact, alias, msg, user)
return False, "550 SL E6"
mailbox = Mailbox.get_by(email=mail_from, user_id=user.id)
if not mailbox or mailbox not in alias.mailboxes:
# only mailbox can send email to the reply-email
handle_unknown_mailbox(envelope, msg, reply_email, user, alias)
return False, "550 SL E7"
if ENFORCE_SPF and mailbox.force_spf:
ip = msg[_IP_HEADER]
if not spf_pass(ip, envelope, mailbox, user, alias,
contact.website_email, msg):
# cannot use 4** here as sender will retry. 5** because that generates bounce report
return True, "250 SL E11"
delete_header(msg, _IP_HEADER)
delete_header(msg, "DKIM-Signature")
delete_header(msg, "Received")
# make the email comes from alias
from_header = alias.email
# add alias name from alias
if alias.name:
LOG.d("Put alias name in from header")
from_header = formataddr((alias.name, alias.email))
elif alias.custom_domain:
LOG.d("Put domain default alias name in from header")
# add alias name from domain
if alias.custom_domain.name:
from_header = formataddr((alias.custom_domain.name, alias.email))
add_or_replace_header(msg, "From", from_header)
# some email providers like ProtonMail adds automatically the Reply-To field
# make sure to delete it
delete_header(msg, "Reply-To")
# remove sender header if present as this could reveal user real email
delete_header(msg, "Sender")
delete_header(msg, "X-Sender")
replace_header_when_reply(msg, alias, "To")
replace_header_when_reply(msg, alias, "Cc")
# Received-SPF is injected by postfix-policyd-spf-python can reveal user original email
delete_header(msg, "Received-SPF")
LOG.d(
"send email from %s to %s, mail_options:%s,rcpt_options:%s",
alias.email,
contact.website_email,
envelope.mail_options,
envelope.rcpt_options,
)
if alias_domain in ALIAS_DOMAINS:
add_dkim_signature(msg, alias_domain)
# add DKIM-Signature for custom-domain alias
else:
custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain)
if custom_domain.dkim_verified:
add_dkim_signature(msg, alias_domain)
smtp.sendmail(
alias.email,
contact.website_email,
msg.as_bytes(),
envelope.mail_options,
envelope.rcpt_options,
)
EmailLog.create(contact_id=contact.id,
is_reply=True,
user_id=contact.user_id)
db.session.commit()
return True, "250 Message accepted for delivery"
def test_get_aliases_v2(flask_client):
user = User.create(email="[email protected]",
password="******",
name="Test User",
activated=True)
db.session.commit()
# create api_key
api_key = ApiKey.create(user.id, "for test")
db.session.commit()
a0 = Alias.create_new(user, "prefix0")
a1 = Alias.create_new(user, "prefix1")
db.session.commit()
# add activity for a0
c0 = Contact.create(
user_id=user.id,
alias_id=a0.id,
website_email="*****@*****.**",
reply_email="re0@SL",
)
db.session.commit()
EmailLog.create(contact_id=c0.id, user_id=user.id)
db.session.commit()
# a1 has more recent activity
c1 = Contact.create(
user_id=user.id,
alias_id=a1.id,
website_email="*****@*****.**",
reply_email="re1@SL",
)
db.session.commit()
EmailLog.create(contact_id=c1.id, user_id=user.id)
db.session.commit()
# get aliases v2
r = flask_client.get(
url_for("api.get_aliases_v2", page_id=0),
headers={"Authentication": api_key.code},
)
assert r.status_code == 200
# make sure a1 is returned before a0
r0 = r.json["aliases"][0]
# r0 will have the following format
# {
# "creation_date": "2020-04-25 21:10:01+00:00",
# "creation_timestamp": 1587849001,
# "email": "*****@*****.**",
# "enabled": true,
# "id": 3,
# "name": "Hey hey",
# "latest_activity": {
# "action": "forward",
# "contact": {
# "email": "*****@*****.**",
# "name": null,
# "reverse_alias": "\"c1 at example.com\" <re1@SL>"
# },
# "timestamp": 1587849001
# },
# "mailbox": {
# "email": "[email protected]",
# "id": 1
# },
# "nb_block": 0,
# "nb_forward": 1,
# "nb_reply": 0,
# "note": null
# }
assert "name" in r0
assert r0["email"].startswith("prefix1")
assert r0["latest_activity"]["action"] == "forward"
assert "timestamp" in r0["latest_activity"]
assert r0["latest_activity"]["contact"]["email"] == "*****@*****.**"
assert "name" in r0["latest_activity"]["contact"]
assert "reverse_alias" in r0["latest_activity"]["contact"]
assert "id" in r0["mailbox"]
assert "email" in r0["mailbox"]
assert r0["mailboxes"]
for mailbox in r0["mailboxes"]:
assert "id" in mailbox
assert "email" in mailbox
def test_get_aliases_v2(flask_client):
user = User.create(email="[email protected]",
password="******",
name="Test User",
activated=True)
db.session.commit()
# create api_key
api_key = ApiKey.create(user.id, "for test")
db.session.commit()
a0 = Alias.create_new(user, "prefix0")
a1 = Alias.create_new(user, "prefix1")
db.session.commit()
# << Aliases have no activity >>
r = flask_client.get(
url_for("api.get_aliases_v2", page_id=0),
headers={"Authentication": api_key.code},
)
assert r.status_code == 200
r0 = r.json["aliases"][0]
assert "name" in r0
# make sure a1 is returned before a0
assert r0["email"].startswith("prefix1")
assert "id" in r0["mailbox"]
assert "email" in r0["mailbox"]
assert r0["mailboxes"]
for mailbox in r0["mailboxes"]:
assert "id" in mailbox
assert "email" in mailbox
assert "support_pgp" in r0
assert not r0["support_pgp"]
assert "disable_pgp" in r0
assert not r0["disable_pgp"]
# << Alias has some activities >>
c0 = Contact.create(
user_id=user.id,
alias_id=a0.id,
website_email="*****@*****.**",
reply_email="re0@SL",
)
db.session.commit()
EmailLog.create(contact_id=c0.id, user_id=user.id)
db.session.commit()
# a1 has more recent activity
c1 = Contact.create(
user_id=user.id,
alias_id=a1.id,
website_email="*****@*****.**",
reply_email="re1@SL",
)
db.session.commit()
EmailLog.create(contact_id=c1.id, user_id=user.id)
db.session.commit()
# get aliases v2
r = flask_client.get(
url_for("api.get_aliases_v2", page_id=0),
headers={"Authentication": api_key.code},
)
assert r.status_code == 200
r0 = r.json["aliases"][0]
assert r0["latest_activity"]["action"] == "forward"
assert "timestamp" in r0["latest_activity"]
assert r0["latest_activity"]["contact"]["email"] == "*****@*****.**"
assert "name" in r0["latest_activity"]["contact"]
assert "reverse_alias" in r0["latest_activity"]["contact"]
assert "pinned" in r0
def fake_data():
LOG.d("create fake data")
# Create a user
user = User.create(
email="*****@*****.**",
name="John Wick",
password="******",
activated=True,
is_admin=True,
# enable_otp=True,
otp_secret="base32secret3232",
intro_shown=True,
fido_uuid=None,
)
user.trial_end = None
Session.commit()
# add a profile picture
file_path = "profile_pic.svg"
s3.upload_from_bytesio(
file_path,
open(os.path.join(ROOT_DIR, "static", "default-icon.svg"), "rb"),
content_type="image/svg",
)
file = File.create(user_id=user.id, path=file_path, commit=True)
user.profile_picture_id = file.id
Session.commit()
# create a bounced email
alias = Alias.create_new_random(user)
Session.commit()
bounce_email_file_path = "bounce.eml"
s3.upload_email_from_bytesio(
bounce_email_file_path,
open(os.path.join(ROOT_DIR, "local_data", "email_tests", "2.eml"),
"rb"),
"download.eml",
)
refused_email = RefusedEmail.create(
path=bounce_email_file_path,
full_report_path=bounce_email_file_path,
user_id=user.id,
commit=True,
)
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
alias_id=contact.alias_id,
refused_email_id=refused_email.id,
bounced=True,
commit=True,
)
LifetimeCoupon.create(code="lifetime-coupon", nb_used=10, commit=True)
Coupon.create(code="coupon", commit=True)
# Create a subscription for user
Subscription.create(
user_id=user.id,
cancel_url="https://checkout.paddle.com/subscription/cancel?user=1234",
update_url="https://checkout.paddle.com/subscription/update?user=1234",
subscription_id="123",
event_time=arrow.now(),
next_bill_date=arrow.now().shift(days=10).date(),
plan=PlanEnum.monthly,
commit=True,
)
CoinbaseSubscription.create(user_id=user.id,
end_at=arrow.now().shift(days=10),
commit=True)
api_key = ApiKey.create(user_id=user.id, name="Chrome")
api_key.code = "code"
api_key = ApiKey.create(user_id=user.id, name="Firefox")
api_key.code = "codeFF"
pgp_public_key = open(get_abs_path("local_data/public-pgp.asc")).read()
m1 = Mailbox.create(
user_id=user.id,
email="*****@*****.**",
verified=True,
pgp_public_key=pgp_public_key,
)
m1.pgp_finger_print = load_public_key(pgp_public_key)
Session.commit()
# [email protected] is in a LOT of data breaches
Alias.create(email="*****@*****.**",
user_id=user.id,
mailbox_id=m1.id)
for i in range(3):
if i % 2 == 0:
a = Alias.create(email=f"e{i}@{FIRST_ALIAS_DOMAIN}",
user_id=user.id,
mailbox_id=m1.id)
else:
a = Alias.create(
email=f"e{i}@{FIRST_ALIAS_DOMAIN}",
user_id=user.id,
mailbox_id=user.default_mailbox_id,
)
Session.commit()
if i % 5 == 0:
if i % 2 == 0:
AliasMailbox.create(alias_id=a.id,
mailbox_id=user.default_mailbox_id)
else:
AliasMailbox.create(alias_id=a.id, mailbox_id=m1.id)
Session.commit()
# some aliases don't have any activity
# if i % 3 != 0:
# contact = Contact.create(
# user_id=user.id,
# alias_id=a.id,
# website_email=f"contact{i}@example.com",
# reply_email=f"rep{i}@sl.local",
# )
# Session.commit()
# for _ in range(3):
# EmailLog.create(user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id)
# Session.commit()
# have some disabled alias
if i % 5 == 0:
a.enabled = False
Session.commit()
custom_domain1 = CustomDomain.create(user_id=user.id,
domain="ab.cd",
verified=True)
Session.commit()
Alias.create(
user_id=user.id,
email="*****@*****.**",
mailbox_id=user.default_mailbox_id,
custom_domain_id=custom_domain1.id,
commit=True,
)
Alias.create(
user_id=user.id,
email="*****@*****.**",
mailbox_id=user.default_mailbox_id,
custom_domain_id=custom_domain1.id,
commit=True,
)
Directory.create(user_id=user.id, name="abcd")
Directory.create(user_id=user.id, name="xyzt")
Session.commit()
# Create a client
client1 = Client.create_new(name="Demo", user_id=user.id)
client1.oauth_client_id = "client-id"
client1.oauth_client_secret = "client-secret"
Session.commit()
RedirectUri.create(client_id=client1.id,
uri="https://your-website.com/oauth-callback")
client2 = Client.create_new(name="Demo 2", user_id=user.id)
client2.oauth_client_id = "client-id2"
client2.oauth_client_secret = "client-secret2"
Session.commit()
ClientUser.create(user_id=user.id, client_id=client1.id, name="Fake Name")
referral = Referral.create(user_id=user.id,
code="Website",
name="First referral")
Referral.create(user_id=user.id, code="Podcast", name="First referral")
Payout.create(user_id=user.id,
amount=1000,
number_upgraded_account=100,
payment_method="BTC")
Payout.create(
user_id=user.id,
amount=5000,
number_upgraded_account=200,
payment_method="PayPal",
)
Session.commit()
for i in range(6):
Notification.create(user_id=user.id,
message=f"""Hey hey <b>{i}</b> """ * 10)
Session.commit()
user2 = User.create(
email="*****@*****.**",
password="******",
activated=True,
referral_id=referral.id,
)
Mailbox.create(user_id=user2.id,
email="*****@*****.**",
verified=True)
Session.commit()
ManualSubscription.create(
user_id=user2.id,
end_at=arrow.now().shift(years=1, days=1),
comment="Local manual",
commit=True,
)
SLDomain.create(domain="premium.com", premium_only=True, commit=True)
hibp1 = Hibp.create(name="first breach",
description="breach description",
commit=True)
hibp2 = Hibp.create(name="second breach",
description="breach description",
commit=True)
breached_alias1 = Alias.create(email="*****@*****.**",
user_id=user.id,
mailbox_id=m1.id,
commit=True)
breached_alias2 = Alias.create(email="*****@*****.**",
user_id=user.id,
mailbox_id=m1.id,
commit=True)
AliasHibp.create(hibp_id=hibp1.id, alias_id=breached_alias1.id)
AliasHibp.create(hibp_id=hibp2.id, alias_id=breached_alias2.id)
# old domain will have ownership_verified=True
CustomDomain.create(user_id=user.id,
domain="old.com",
verified=True,
ownership_verified=True)
