def handle_email_sent_to_ourself(alias, mailbox, msg: Message, user):
# store the refused email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/cycle-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()),
random_name)
refused_email = RefusedEmail.create(path=None,
full_report_path=full_report_path,
user_id=alias.user_id)
db.session.commit()
LOG.d("Create refused email %s", refused_email)
# link available for 6 days as it gets deleted in 7 days
refused_email_url = refused_email.get_url(expires_in=518400)
send_email_with_rate_control(
user,
ALERT_SEND_EMAIL_CYCLE,
mailbox.email,
f"Warning: email sent from {mailbox.email} to {alias.email} forms a cycle",
render(
"transactional/cycle-email.txt",
name=user.name or "",
alias=alias,
mailbox=mailbox,
refused_email_url=refused_email_url,
),
render(
"transactional/cycle-email.html",
name=user.name or "",
alias=alias,
mailbox=mailbox,
refused_email_url=refused_email_url,
),
)
def delete_refused_emails():
for refused_email in RefusedEmail.filter_by(deleted=False).all():
if arrow.now().shift(days=1) > refused_email.delete_at >= arrow.now():
LOG.d("Delete refused email %s", refused_email)
if refused_email.path:
s3.delete(refused_email.path)
s3.delete(refused_email.full_report_path)
# do not set path and full_report_path to null
# so we can check later that the files are indeed deleted
refused_email.deleted = True
Session.commit()
LOG.d("Finish delete_refused_emails")
def fake_data():
LOG.d("create fake data")
# Remove db if exist
if os.path.exists("db.sqlite"):
LOG.d("remove existing db file")
os.remove("db.sqlite")
# Create all tables
db.create_all()
# Create a user
user = User.create(
email="*****@*****.**",
name="John Wick",
password="******",
activated=True,
is_admin=True,
enable_otp=False,
otp_secret="base32secret3232",
intro_shown=True,
fido_uuid=None,
)
user.include_sender_in_reverse_alias = None
user.trial_end = None
db.session.commit()
# add a profile picture
file_path = "profile_pic.svg"
s3.upload_from_bytesio(
file_path,
open(os.path.join(ROOT_DIR, "static", "default-icon.svg"), "rb"),
content_type="image/svg",
)
file = File.create(user_id=user.id, path=file_path, commit=True)
user.profile_picture_id = file.id
db.session.commit()
# create a bounced email
alias = Alias.create_new_random(user)
db.session.commit()
bounce_email_file_path = "bounce.eml"
s3.upload_email_from_bytesio(
bounce_email_file_path,
open(os.path.join(ROOT_DIR, "local_data", "email_tests", "2.eml"),
"rb"),
"download.eml",
)
refused_email = RefusedEmail.create(
path=bounce_email_file_path,
full_report_path=bounce_email_file_path,
user_id=user.id,
commit=True,
)
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
refused_email_id=refused_email.id,
bounced=True,
commit=True,
)
LifetimeCoupon.create(code="coupon", nb_used=10, commit=True)
# Create a subscription for user
Subscription.create(
user_id=user.id,
cancel_url="https://checkout.paddle.com/subscription/cancel?user=1234",
update_url="https://checkout.paddle.com/subscription/update?user=1234",
subscription_id="123",
event_time=arrow.now(),
next_bill_date=arrow.now().shift(days=10).date(),
plan=PlanEnum.monthly,
commit=True,
)
CoinbaseSubscription.create(user_id=user.id,
end_at=arrow.now().shift(days=10),
commit=True)
api_key = ApiKey.create(user_id=user.id, name="Chrome")
api_key.code = "code"
api_key = ApiKey.create(user_id=user.id, name="Firefox")
api_key.code = "codeFF"
pgp_public_key = open(get_abs_path("local_data/public-pgp.asc")).read()
m1 = Mailbox.create(
user_id=user.id,
email="*****@*****.**",
verified=True,
pgp_public_key=pgp_public_key,
)
m1.pgp_finger_print = load_public_key(pgp_public_key)
db.session.commit()
for i in range(3):
if i % 2 == 0:
a = Alias.create(email=f"e{i}@{FIRST_ALIAS_DOMAIN}",
user_id=user.id,
mailbox_id=m1.id)
else:
a = Alias.create(
email=f"e{i}@{FIRST_ALIAS_DOMAIN}",
user_id=user.id,
mailbox_id=user.default_mailbox_id,
)
db.session.commit()
if i % 5 == 0:
if i % 2 == 0:
AliasMailbox.create(alias_id=a.id,
mailbox_id=user.default_mailbox_id)
else:
AliasMailbox.create(alias_id=a.id, mailbox_id=m1.id)
db.session.commit()
# some aliases don't have any activity
# if i % 3 != 0:
# contact = Contact.create(
# user_id=user.id,
# alias_id=a.id,
# website_email=f"contact{i}@example.com",
# reply_email=f"rep{i}@sl.local",
# )
# db.session.commit()
# for _ in range(3):
# EmailLog.create(user_id=user.id, contact_id=contact.id)
# db.session.commit()
# have some disabled alias
if i % 5 == 0:
a.enabled = False
db.session.commit()
CustomDomain.create(user_id=user.id, domain="ab.cd", verified=True)
CustomDomain.create(user_id=user.id,
domain="very-long-domain.com.net.org",
verified=True)
db.session.commit()
Directory.create(user_id=user.id, name="abcd")
Directory.create(user_id=user.id, name="xyzt")
db.session.commit()
# Create a client
client1 = Client.create_new(name="Demo", user_id=user.id)
client1.oauth_client_id = "client-id"
client1.oauth_client_secret = "client-secret"
client1.published = True
db.session.commit()
RedirectUri.create(client_id=client1.id, uri="https://ab.com")
client2 = Client.create_new(name="Demo 2", user_id=user.id)
client2.oauth_client_id = "client-id2"
client2.oauth_client_secret = "client-secret2"
client2.published = True
db.session.commit()
ClientUser.create(user_id=user.id, client_id=client1.id, name="Fake Name")
referral = Referral.create(user_id=user.id,
code="REFCODE",
name="First referral")
db.session.commit()
for i in range(6):
Notification.create(user_id=user.id,
message=f"""Hey hey <b>{i}</b> """ * 10)
db.session.commit()
User.create(
email="*****@*****.**",
password="******",
activated=True,
referral_id=referral.id,
)
db.session.commit()
def handle_spam(
contact: Contact,
alias: Alias,
msg: Message,
user: User,
mailbox_email: str,
email_log: EmailLog,
):
# Store the report & original email
orig_msg = get_orig_message_from_spamassassin_report(msg)
# generate a name for the email
random_name = str(uuid.uuid4())
full_report_path = f"spams/full-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()),
random_name)
file_path = None
if orig_msg:
file_path = f"spams/{random_name}.eml"
s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()),
random_name)
refused_email = RefusedEmail.create(path=file_path,
full_report_path=full_report_path,
user_id=user.id)
db.session.flush()
email_log.refused_email_id = refused_email.id
db.session.commit()
LOG.d("Create spam email %s", refused_email)
refused_email_url = (URL + f"/dashboard/refused_email?highlight_id=" +
str(email_log.id))
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
# inform user
LOG.d(
"Inform user %s about spam email sent by %s to alias %s",
user,
contact.website_email,
alias.email,
)
send_email_with_rate_control(
user,
ALERT_SPAM_EMAIL,
mailbox_email,
f"Email from {contact.website_email} to {alias.email} is detected as spam",
render(
"transactional/spam-email.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
),
render(
"transactional/spam-email.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
),
)
def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User):
address = alias.email
email_log: EmailLog = EmailLog.create(contact_id=contact.id,
bounced=True,
user_id=contact.user_id)
db.session.commit()
nb_bounced = EmailLog.filter_by(contact_id=contact.id,
bounced=True).count()
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
# <<< Store the bounced email >>>
# generate a name for the email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/full-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()),
random_name)
orig_msg = get_orig_message_from_bounce(msg)
if not orig_msg:
LOG.error(
"Cannot parse original message from bounce message %s %s %s %s",
alias,
user,
contact,
full_report_path,
)
return
file_path = f"refused-emails/{random_name}.eml"
s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()),
random_name)
# <<< END Store the bounced email >>>
mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER])
mailbox = Mailbox.get(mailbox_id)
if not mailbox or mailbox.user_id != user.id:
LOG.error(
"Tampered message mailbox_id %s, %s, %s, %s %s",
mailbox_id,
user,
alias,
contact,
full_report_path,
)
return
refused_email = RefusedEmail.create(path=file_path,
full_report_path=full_report_path,
user_id=user.id)
db.session.flush()
email_log.refused_email_id = refused_email.id
email_log.bounced_mailbox_id = mailbox.id
db.session.commit()
LOG.d("Create refused email %s", refused_email)
refused_email_url = (URL + f"/dashboard/refused_email?highlight_id=" +
str(email_log.id))
# inform user if this is the first bounced email
if nb_bounced == 1:
LOG.d(
"Inform user %s about bounced email sent by %s to alias %s",
user,
contact.website_email,
address,
)
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
# use user mail here as only user is authenticated to see the refused email
user.email,
f"Email from {contact.website_email} to {address} cannot be delivered to your inbox",
render(
"transactional/bounced-email.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/bounced-email.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
# cannot include bounce email as it can contain spammy text
# bounced_email=msg,
)
# disable the alias the second time email is bounced
elif nb_bounced >= 2:
LOG.d(
"Bounce happens again with alias %s from %s. Disable alias now ",
address,
contact.website_email,
)
alias.enabled = False
db.session.commit()
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
# use user mail here as only user is authenticated to see the refused email
user.email,
f"Alias {address} has been disabled due to second undelivered email from {contact.website_email}",
render(
"transactional/automatic-disable-alias.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/automatic-disable-alias.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
# cannot include bounce email as it can contain spammy text
# bounced_email=msg,
)
def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User):
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
# Store the bounced email
# generate a name for the email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/full-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()),
random_name)
file_path = None
mailbox = None
email_log: EmailLog = None
orig_msg = get_orig_message_from_bounce(msg)
if not orig_msg:
# Some MTA does not return the original message in bounce message
# nothing we can do here
LOG.warning(
"Cannot parse original message from bounce message %s %s %s %s",
alias,
user,
contact,
full_report_path,
)
else:
file_path = f"refused-emails/{random_name}.eml"
s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()),
random_name)
try:
mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER])
except TypeError:
LOG.exception(
"cannot parse mailbox from original message header %s",
orig_msg[_MAILBOX_ID_HEADER],
)
else:
mailbox = Mailbox.get(mailbox_id)
if not mailbox or mailbox.user_id != user.id:
LOG.exception(
"Tampered message mailbox_id %s, %s, %s, %s %s",
mailbox_id,
user,
alias,
contact,
full_report_path,
)
# cannot use this tampered mailbox, reset it
mailbox = None
# try to get the original email_log
try:
email_log_id = int(orig_msg[_EMAIL_LOG_ID_HEADER])
except TypeError:
LOG.exception(
"cannot parse email log from original message header %s",
orig_msg[_EMAIL_LOG_ID_HEADER],
)
else:
email_log = EmailLog.get(email_log_id)
refused_email = RefusedEmail.create(path=file_path,
full_report_path=full_report_path,
user_id=user.id)
db.session.flush()
LOG.d("Create refused email %s", refused_email)
if not mailbox:
LOG.debug("Try to get mailbox from bounce report")
try:
mailbox_id = int(get_header_from_bounce(msg, _MAILBOX_ID_HEADER))
except Exception:
LOG.exception("cannot get mailbox-id from bounce report, %s",
refused_email)
else:
mailbox = Mailbox.get(mailbox_id)
if not mailbox or mailbox.user_id != user.id:
LOG.exception(
"Tampered message mailbox_id %s, %s, %s, %s %s",
mailbox_id,
user,
alias,
contact,
full_report_path,
)
mailbox = None
if not email_log:
LOG.d("Try to get email log from bounce report")
try:
email_log_id = int(
get_header_from_bounce(msg, _EMAIL_LOG_ID_HEADER))
except Exception:
LOG.exception("cannot get email log id from bounce report, %s",
refused_email)
else:
email_log = EmailLog.get(email_log_id)
# use the default mailbox as the last option
if not mailbox:
LOG.warning("Use %s default mailbox %s", alias, refused_email)
mailbox = alias.mailbox
# create a new email log as the last option
if not email_log:
LOG.warning("cannot get the original email_log, create a new one")
email_log: EmailLog = EmailLog.create(contact_id=contact.id,
user_id=contact.user_id)
email_log.bounced = True
email_log.refused_email_id = refused_email.id
email_log.bounced_mailbox_id = mailbox.id
db.session.commit()
refused_email_url = (URL + f"/dashboard/refused_email?highlight_id=" +
str(email_log.id))
nb_bounced = EmailLog.filter_by(contact_id=contact.id,
bounced=True).count()
if nb_bounced >= 2 and alias.cannot_be_disabled:
LOG.warning("%s cannot be disabled", alias)
# inform user if this is the first bounced email
if nb_bounced == 1 or (nb_bounced >= 2 and alias.cannot_be_disabled):
LOG.d(
"Inform user %s about bounced email sent by %s to alias %s",
user,
contact.website_email,
alias,
)
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
user.email,
f"Email from {contact.website_email} to {alias.email} cannot be delivered to your inbox",
render(
"transactional/bounced-email.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/bounced-email.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
)
# disable the alias the second time email is bounced
elif nb_bounced >= 2:
LOG.d(
"Bounce happens again with alias %s from %s. Disable alias now ",
alias,
contact.website_email,
)
alias.enabled = False
db.session.commit()
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
user.email,
f"Alias {alias.email} has been disabled due to second undelivered email from {contact.website_email}",
render(
"transactional/automatic-disable-alias.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/automatic-disable-alias.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
)
def fake_data():
LOG.d("create fake data")
# Create a user
user = User.create(
email="*****@*****.**",
name="John Wick",
password="******",
activated=True,
is_admin=True,
# enable_otp=True,
otp_secret="base32secret3232",
intro_shown=True,
fido_uuid=None,
)
user.trial_end = None
Session.commit()
# add a profile picture
file_path = "profile_pic.svg"
s3.upload_from_bytesio(
file_path,
open(os.path.join(ROOT_DIR, "static", "default-icon.svg"), "rb"),
content_type="image/svg",
)
file = File.create(user_id=user.id, path=file_path, commit=True)
user.profile_picture_id = file.id
Session.commit()
# create a bounced email
alias = Alias.create_new_random(user)
Session.commit()
bounce_email_file_path = "bounce.eml"
s3.upload_email_from_bytesio(
bounce_email_file_path,
open(os.path.join(ROOT_DIR, "local_data", "email_tests", "2.eml"),
"rb"),
"download.eml",
)
refused_email = RefusedEmail.create(
path=bounce_email_file_path,
full_report_path=bounce_email_file_path,
user_id=user.id,
commit=True,
)
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
alias_id=contact.alias_id,
refused_email_id=refused_email.id,
bounced=True,
commit=True,
)
LifetimeCoupon.create(code="lifetime-coupon", nb_used=10, commit=True)
Coupon.create(code="coupon", commit=True)
# Create a subscription for user
Subscription.create(
user_id=user.id,
cancel_url="https://checkout.paddle.com/subscription/cancel?user=1234",
update_url="https://checkout.paddle.com/subscription/update?user=1234",
subscription_id="123",
event_time=arrow.now(),
next_bill_date=arrow.now().shift(days=10).date(),
plan=PlanEnum.monthly,
commit=True,
)
CoinbaseSubscription.create(user_id=user.id,
end_at=arrow.now().shift(days=10),
commit=True)
api_key = ApiKey.create(user_id=user.id, name="Chrome")
api_key.code = "code"
api_key = ApiKey.create(user_id=user.id, name="Firefox")
api_key.code = "codeFF"
pgp_public_key = open(get_abs_path("local_data/public-pgp.asc")).read()
m1 = Mailbox.create(
user_id=user.id,
email="*****@*****.**",
verified=True,
pgp_public_key=pgp_public_key,
)
m1.pgp_finger_print = load_public_key(pgp_public_key)
Session.commit()
# [email protected] is in a LOT of data breaches
Alias.create(email="*****@*****.**",
user_id=user.id,
mailbox_id=m1.id)
for i in range(3):
if i % 2 == 0:
a = Alias.create(email=f"e{i}@{FIRST_ALIAS_DOMAIN}",
user_id=user.id,
mailbox_id=m1.id)
else:
a = Alias.create(
email=f"e{i}@{FIRST_ALIAS_DOMAIN}",
user_id=user.id,
mailbox_id=user.default_mailbox_id,
)
Session.commit()
if i % 5 == 0:
if i % 2 == 0:
AliasMailbox.create(alias_id=a.id,
mailbox_id=user.default_mailbox_id)
else:
AliasMailbox.create(alias_id=a.id, mailbox_id=m1.id)
Session.commit()
# some aliases don't have any activity
# if i % 3 != 0:
# contact = Contact.create(
# user_id=user.id,
# alias_id=a.id,
# website_email=f"contact{i}@example.com",
# reply_email=f"rep{i}@sl.local",
# )
# Session.commit()
# for _ in range(3):
# EmailLog.create(user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id)
# Session.commit()
# have some disabled alias
if i % 5 == 0:
a.enabled = False
Session.commit()
custom_domain1 = CustomDomain.create(user_id=user.id,
domain="ab.cd",
verified=True)
Session.commit()
Alias.create(
user_id=user.id,
email="*****@*****.**",
mailbox_id=user.default_mailbox_id,
custom_domain_id=custom_domain1.id,
commit=True,
)
Alias.create(
user_id=user.id,
email="*****@*****.**",
mailbox_id=user.default_mailbox_id,
custom_domain_id=custom_domain1.id,
commit=True,
)
Directory.create(user_id=user.id, name="abcd")
Directory.create(user_id=user.id, name="xyzt")
Session.commit()
# Create a client
client1 = Client.create_new(name="Demo", user_id=user.id)
client1.oauth_client_id = "client-id"
client1.oauth_client_secret = "client-secret"
Session.commit()
RedirectUri.create(client_id=client1.id,
uri="https://your-website.com/oauth-callback")
client2 = Client.create_new(name="Demo 2", user_id=user.id)
client2.oauth_client_id = "client-id2"
client2.oauth_client_secret = "client-secret2"
Session.commit()
ClientUser.create(user_id=user.id, client_id=client1.id, name="Fake Name")
referral = Referral.create(user_id=user.id,
code="Website",
name="First referral")
Referral.create(user_id=user.id, code="Podcast", name="First referral")
Payout.create(user_id=user.id,
amount=1000,
number_upgraded_account=100,
payment_method="BTC")
Payout.create(
user_id=user.id,
amount=5000,
number_upgraded_account=200,
payment_method="PayPal",
)
Session.commit()
for i in range(6):
Notification.create(user_id=user.id,
message=f"""Hey hey <b>{i}</b> """ * 10)
Session.commit()
user2 = User.create(
email="*****@*****.**",
password="******",
activated=True,
referral_id=referral.id,
)
Mailbox.create(user_id=user2.id,
email="*****@*****.**",
verified=True)
Session.commit()
ManualSubscription.create(
user_id=user2.id,
end_at=arrow.now().shift(years=1, days=1),
comment="Local manual",
commit=True,
)
SLDomain.create(domain="premium.com", premium_only=True, commit=True)
hibp1 = Hibp.create(name="first breach",
description="breach description",
commit=True)
hibp2 = Hibp.create(name="second breach",
description="breach description",
commit=True)
breached_alias1 = Alias.create(email="*****@*****.**",
user_id=user.id,
mailbox_id=m1.id,
commit=True)
breached_alias2 = Alias.create(email="*****@*****.**",
user_id=user.id,
mailbox_id=m1.id,
commit=True)
AliasHibp.create(hibp_id=hibp1.id, alias_id=breached_alias1.id)
AliasHibp.create(hibp_id=hibp2.id, alias_id=breached_alias2.id)
# old domain will have ownership_verified=True
CustomDomain.create(user_id=user.id,
domain="old.com",
verified=True,
ownership_verified=True)