def sanity_check(): """ #TODO: investigate why DNS sometimes not working Different sanity checks - detect if there's mailbox that's using a invalid domain """ mailbox_ids = ( db.session.query(Mailbox.id) .filter(Mailbox.verified.is_(True), Mailbox.disabled.is_(False)) .all() ) mailbox_ids = [e[0] for e in mailbox_ids] # iterate over id instead of mailbox directly # as a mailbox can be deleted during the sleep time for mailbox_id in mailbox_ids: mailbox = Mailbox.get(mailbox_id) # a mailbox has been deleted if not mailbox: continue # hack to not query DNS too often sleep(1) if not email_can_be_used_as_mailbox(mailbox.email): mailbox.nb_failed_checks += 1 nb_email_log = nb_email_log_for_mailbox(mailbox) # send a warning if mailbox.nb_failed_checks == 5: if mailbox.user.email != mailbox.email: send_email( mailbox.user.email, f"Mailbox {mailbox.email} is disabled", render( "transactional/disable-mailbox-warning.txt", mailbox=mailbox ), render( "transactional/disable-mailbox-warning.html", mailbox=mailbox, ), ) # alert if too much fail and nb_email_log > 100 if mailbox.nb_failed_checks > 10 and nb_email_log > 100: mailbox.disabled = True if mailbox.user.email != mailbox.email: send_email( mailbox.user.email, f"Mailbox {mailbox.email} is disabled", render("transactional/disable-mailbox.txt", mailbox=mailbox), render("transactional/disable-mailbox.html", mailbox=mailbox), ) LOG.warning( "issue with mailbox %s domain. #alias %s, nb email log %s", mailbox, mailbox.nb_alias(), nb_email_log, ) else: # reset nb check mailbox.nb_failed_checks = 0 db.session.commit() for user in User.filter_by(activated=True).all(): if sanitize_email(user.email) != user.email: LOG.exception("%s does not have sanitized email", user) for alias in Alias.query.all(): if sanitize_email(alias.email) != alias.email: LOG.exception("Alias %s email not sanitized", alias) if alias.name and "\n" in alias.name: alias.name = alias.name.replace("\n", "") db.session.commit() LOG.exception("Alias %s name contains linebreak %s", alias, alias.name) contact_email_sanity_date = arrow.get("2021-01-12") for contact in Contact.query.all(): if sanitize_email(contact.reply_email) != contact.reply_email: LOG.exception("Contact %s reply-email not sanitized", contact) if ( sanitize_email(contact.website_email) != contact.website_email and contact.created_at > contact_email_sanity_date ): LOG.exception("Contact %s website-email not sanitized", contact) if not contact.invalid_email and not is_valid_email(contact.website_email): LOG.exception("%s invalid email", contact) contact.invalid_email = True db.session.commit() for mailbox in Mailbox.query.all(): if sanitize_email(mailbox.email) != mailbox.email: LOG.exception("Mailbox %s address not sanitized", mailbox) for contact in Contact.query.all(): if normalize_reply_email(contact.reply_email) != contact.reply_email: LOG.exception( "Contact %s reply email is not normalized %s", contact, contact.reply_email, ) for domain in CustomDomain.query.all(): if domain.name and "\n" in domain.name: LOG.exception("Domain %s name contain linebreak %s", domain, domain.name) LOG.d("Finish sanity check")
def handle_reply(self, envelope, smtp: SMTP, msg: Message) -> str: reply_email = envelope.rcpt_tos[0].lower() # reply_email must end with EMAIL_DOMAIN if not reply_email.endswith(EMAIL_DOMAIN): LOG.error(f"Reply email {reply_email} has wrong domain") return "550 wrong reply email" forward_email = ForwardEmail.get_by(reply_email=reply_email) alias: str = forward_email.gen_email.email # alias must end with EMAIL_DOMAIN or custom-domain alias_domain = alias[alias.find("@") + 1 :] if alias_domain != EMAIL_DOMAIN: if not CustomDomain.get_by(domain=alias_domain): return "550 alias unknown by SimpleLogin" user_email = forward_email.gen_email.user.email if envelope.mail_from.lower() != user_email.lower(): LOG.error( f"Reply email can only be used by user email. Actual mail_from: %s. User email %s. reply_email %s", envelope.mail_from, user_email, reply_email, ) send_email( envelope.mail_from, f"Your email ({envelope.mail_from}) is not allowed to send email to {reply_email}", "", "", ) return "550 ignored" delete_header(msg, "DKIM-Signature") # the email comes from alias msg.replace_header("From", alias) # some email providers like ProtonMail adds automatically the Reply-To field # make sure to delete it delete_header(msg, "Reply-To") msg.replace_header("To", forward_email.website_email) # add List-Unsubscribe header unsubscribe_link = f"{URL}/dashboard/unsubscribe/{forward_email.gen_email_id}" add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>") add_or_replace_header( msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click" ) # Received-SPF is injected by postfix-policyd-spf-python can reveal user original email delete_header(msg, "Received-SPF") LOG.d( "send email from %s to %s, mail_options:%s,rcpt_options:%s", alias, forward_email.website_email, envelope.mail_options, envelope.rcpt_options, ) if alias_domain == EMAIL_DOMAIN: add_dkim_signature(msg, EMAIL_DOMAIN) # add DKIM-Signature for non-custom-domain alias else: custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain) if custom_domain.dkim_verified: add_dkim_signature(msg, alias_domain) msg_raw = msg.as_string().encode() smtp.sendmail( alias, forward_email.website_email, msg_raw, envelope.mail_options, envelope.rcpt_options, ) ForwardEmailLog.create(forward_id=forward_email.id, is_reply=True) db.session.commit() return "250 Message accepted for delivery"
def notify_manual_sub_end(): for manual_sub in ManualSubscription.query.all(): need_reminder = False if arrow.now().shift(days=14) > manual_sub.end_at > arrow.now().shift(days=13): need_reminder = True elif arrow.now().shift(days=4) > manual_sub.end_at > arrow.now().shift(days=3): need_reminder = True if need_reminder: user = manual_sub.user LOG.debug("Remind user %s that their manual sub is ending soon", user) send_email( user.email, f"Your subscription will end soon", render( "transactional/manual-subscription-end.txt", user=user, manual_sub=manual_sub, ), render( "transactional/manual-subscription-end.html", user=user, manual_sub=manual_sub, ), ) extend_subscription_url = URL + "/dashboard/coinbase_checkout" for coinbase_subscription in CoinbaseSubscription.query.all(): need_reminder = False if ( arrow.now().shift(days=14) > coinbase_subscription.end_at > arrow.now().shift(days=13) ): need_reminder = True elif ( arrow.now().shift(days=4) > coinbase_subscription.end_at > arrow.now().shift(days=3) ): need_reminder = True if need_reminder: user = coinbase_subscription.user LOG.debug( "Remind user %s that their coinbase subscription is ending soon", user ) send_email( user.email, "Your SimpleLogin subscription will end soon", render( "transactional/coinbase/reminder-subscription.txt", coinbase_subscription=coinbase_subscription, extend_subscription_url=extend_subscription_url, ), render( "transactional/coinbase/reminder-subscription.html", coinbase_subscription=coinbase_subscription, extend_subscription_url=extend_subscription_url, ), )
def stats(): """send admin stats everyday""" if not ADMIN_EMAIL: LOG.w("ADMIN_EMAIL not set, nothing to do") return stats_today = compute_metric2() stats_yesterday = (Metric2.filter( Metric2.date < stats_today.date).order_by(Metric2.date.desc()).first()) today = arrow.now().format() growth_stats = f""" Growth Stats for {today} nb_user: {stats_today.nb_user} - {increase_percent(stats_yesterday.nb_user, stats_today.nb_user)} nb_premium: {stats_today.nb_premium} - {increase_percent(stats_yesterday.nb_premium, stats_today.nb_premium)} nb_cancelled_premium: {stats_today.nb_cancelled_premium} - {increase_percent(stats_yesterday.nb_cancelled_premium, stats_today.nb_cancelled_premium)} nb_apple_premium: {stats_today.nb_apple_premium} - {increase_percent(stats_yesterday.nb_apple_premium, stats_today.nb_apple_premium)} nb_manual_premium: {stats_today.nb_manual_premium} - {increase_percent(stats_yesterday.nb_manual_premium, stats_today.nb_manual_premium)} nb_coinbase_premium: {stats_today.nb_coinbase_premium} - {increase_percent(stats_yesterday.nb_coinbase_premium, stats_today.nb_coinbase_premium)} nb_alias: {stats_today.nb_alias} - {increase_percent(stats_yesterday.nb_alias, stats_today.nb_alias)} nb_forward_last_24h: {stats_today.nb_forward_last_24h} - {increase_percent(stats_yesterday.nb_forward_last_24h, stats_today.nb_forward_last_24h)} nb_reply_last_24h: {stats_today.nb_reply_last_24h} - {increase_percent(stats_yesterday.nb_reply_last_24h, stats_today.nb_reply_last_24h)} nb_block_last_24h: {stats_today.nb_block_last_24h} - {increase_percent(stats_yesterday.nb_block_last_24h, stats_today.nb_block_last_24h)} nb_bounced_last_24h: {stats_today.nb_bounced_last_24h} - {increase_percent(stats_yesterday.nb_bounced_last_24h, stats_today.nb_bounced_last_24h)} nb_custom_domain: {stats_today.nb_verified_custom_domain} - {increase_percent(stats_yesterday.nb_verified_custom_domain, stats_today.nb_verified_custom_domain)} nb_subdomain: {stats_today.nb_subdomain} - {increase_percent(stats_yesterday.nb_subdomain, stats_today.nb_subdomain)} nb_directory: {stats_today.nb_directory} - {increase_percent(stats_yesterday.nb_directory, stats_today.nb_directory)} nb_deleted_directory: {stats_today.nb_deleted_directory} - {increase_percent(stats_yesterday.nb_deleted_directory, stats_today.nb_deleted_directory)} nb_deleted_subdomain: {stats_today.nb_deleted_subdomain} - {increase_percent(stats_yesterday.nb_deleted_subdomain, stats_today.nb_deleted_subdomain)} nb_app: {stats_today.nb_app} - {increase_percent(stats_yesterday.nb_app, stats_today.nb_app)} nb_referred_user: {stats_today.nb_referred_user} - {increase_percent(stats_yesterday.nb_referred_user, stats_today.nb_referred_user)} nb_referred_user_upgrade: {stats_today.nb_referred_user_paid} - {increase_percent(stats_yesterday.nb_referred_user_paid, stats_today.nb_referred_user_paid)} """ LOG.d("growth_stats email: %s", growth_stats) send_email( ADMIN_EMAIL, subject=f"SimpleLogin Growth Stats for {today}", plaintext=growth_stats, retries=3, ) monitoring_report = f""" Monitoring Stats for {today} nb_alias: {stats_today.nb_alias} - {increase_percent(stats_yesterday.nb_alias, stats_today.nb_alias)} nb_forward_last_24h: {stats_today.nb_forward_last_24h} - {increase_percent(stats_yesterday.nb_forward_last_24h, stats_today.nb_forward_last_24h)} nb_reply_last_24h: {stats_today.nb_reply_last_24h} - {increase_percent(stats_yesterday.nb_reply_last_24h, stats_today.nb_reply_last_24h)} nb_block_last_24h: {stats_today.nb_block_last_24h} - {increase_percent(stats_yesterday.nb_block_last_24h, stats_today.nb_block_last_24h)} nb_bounced_last_24h: {stats_today.nb_bounced_last_24h} - {increase_percent(stats_yesterday.nb_bounced_last_24h, stats_today.nb_bounced_last_24h)} nb_total_bounced_last_24h: {stats_today.nb_total_bounced_last_24h} - {increase_percent(stats_yesterday.nb_total_bounced_last_24h, stats_today.nb_total_bounced_last_24h)} """ monitoring_report += "\n====================================\n" monitoring_report += f""" # Account bounce report: """ for email, bounces in bounce_report(): monitoring_report += f"{email}: {bounces}\n" monitoring_report += f"""\n # Alias creation report: """ for email, nb_alias, date in alias_creation_report(): monitoring_report += f"{email}, {date}: {nb_alias}\n" monitoring_report += f"""\n # Full bounce detail report: """ monitoring_report += all_bounce_report() LOG.d("monitoring_report email: %s", monitoring_report) send_email( MONITORING_EMAIL, subject=f"SimpleLogin Monitoring Report for {today}", plaintext=monitoring_report, retries=3, )
def paddle(): LOG.debug( f"paddle callback {request.form.get('alert_name')} {request.form}") # make sure the request comes from Paddle if not paddle_utils.verify_incoming_request(dict(request.form)): LOG.exception("request not coming from paddle. Request data:%s", dict(request.form)) return "KO", 400 if (request.form.get("alert_name") == "subscription_created" ): # new user subscribes # the passthrough is json encoded, e.g. # request.form.get("passthrough") = '{"user_id": 88 }' passthrough = json.loads(request.form.get("passthrough")) user_id = passthrough.get("user_id") user = User.get(user_id) if (int(request.form.get("subscription_plan_id")) == PADDLE_MONTHLY_PRODUCT_ID): plan = PlanEnum.monthly else: plan = PlanEnum.yearly sub = Subscription.get_by(user_id=user.id) if not sub: LOG.d(f"create a new Subscription for user {user}") Subscription.create( user_id=user.id, cancel_url=request.form.get("cancel_url"), update_url=request.form.get("update_url"), subscription_id=request.form.get("subscription_id"), event_time=arrow.now(), next_bill_date=arrow.get( request.form.get("next_bill_date"), "YYYY-MM-DD").date(), plan=plan, ) else: LOG.d(f"Update an existing Subscription for user {user}") sub.cancel_url = request.form.get("cancel_url") sub.update_url = request.form.get("update_url") sub.subscription_id = request.form.get("subscription_id") sub.event_time = arrow.now() sub.next_bill_date = arrow.get( request.form.get("next_bill_date"), "YYYY-MM-DD").date() sub.plan = plan # make sure to set the new plan as not-cancelled # in case user cancels a plan and subscribes a new plan sub.cancelled = False LOG.debug("User %s upgrades!", user) db.session.commit() elif request.form.get( "alert_name") == "subscription_payment_succeeded": subscription_id = request.form.get("subscription_id") LOG.debug("Update subscription %s", subscription_id) sub: Subscription = Subscription.get_by( subscription_id=subscription_id) # when user subscribes, the "subscription_payment_succeeded" can arrive BEFORE "subscription_created" # at that time, subscription object does not exist yet if sub: sub.event_time = arrow.now() sub.next_bill_date = arrow.get( request.form.get("next_bill_date"), "YYYY-MM-DD").date() db.session.commit() elif request.form.get("alert_name") == "subscription_cancelled": subscription_id = request.form.get("subscription_id") sub: Subscription = Subscription.get_by( subscription_id=subscription_id) if sub: # cancellation_effective_date should be the same as next_bill_date LOG.warning( "Cancel subscription %s %s on %s, next bill date %s", subscription_id, sub.user, request.form.get("cancellation_effective_date"), sub.next_bill_date, ) sub.event_time = arrow.now() sub.cancelled = True db.session.commit() user = sub.user send_email( user.email, f"SimpleLogin - what can we do to improve the product?", render( "transactional/subscription-cancel.txt", name=user.name or "", end_date=request.form.get( "cancellation_effective_date"), ), ) else: return "No such subscription", 400 elif request.form.get("alert_name") == "subscription_updated": subscription_id = request.form.get("subscription_id") sub: Subscription = Subscription.get_by( subscription_id=subscription_id) if sub: LOG.debug( "Update subscription %s %s on %s, next bill date %s", subscription_id, sub.user, request.form.get("cancellation_effective_date"), sub.next_bill_date, ) if (int(request.form.get("subscription_plan_id")) == PADDLE_MONTHLY_PRODUCT_ID): plan = PlanEnum.monthly else: plan = PlanEnum.yearly sub.cancel_url = request.form.get("cancel_url") sub.update_url = request.form.get("update_url") sub.event_time = arrow.now() sub.next_bill_date = arrow.get( request.form.get("next_bill_date"), "YYYY-MM-DD").date() sub.plan = plan # make sure to set the new plan as not-cancelled sub.cancelled = False db.session.commit() else: return "No such subscription", 400 return "OK"
def handle_bounce( alias, envelope, forward_email, gen_email, msg, smtp, user, mailbox_email ): ForwardEmailLog.create(forward_id=forward_email.id, bounced=True) db.session.commit() nb_bounced = ForwardEmailLog.filter_by( forward_id=forward_email.id, bounced=True ).count() disable_alias_link = f"{URL}/dashboard/unsubscribe/{gen_email.id}" # inform user if this is the first bounced email if nb_bounced == 1: LOG.d( "Inform user %s about bounced email sent by %s to alias %s", user, forward_email.website_from, alias, ) send_email( mailbox_email, f"Email from {forward_email.website_from} to {alias} cannot be delivered to your inbox", render( "transactional/bounced-email.txt", name=user.name, alias=alias, website_from=forward_email.website_from, website_email=forward_email.website_email, disable_alias_link=disable_alias_link, ), render( "transactional/bounced-email.html", name=user.name, alias=alias, website_from=forward_email.website_from, website_email=forward_email.website_email, disable_alias_link=disable_alias_link, ), bounced_email=msg, ) # disable the alias the second time email is bounced elif nb_bounced >= 2: LOG.d( "Bounce happens again with alias %s from %s. Disable alias now ", alias, forward_email.website_from, ) gen_email.enabled = False db.session.commit() send_email( mailbox_email, f"Alias {alias} has been disabled due to second undelivered email from {forward_email.website_from}", render( "transactional/automatic-disable-alias.txt", name=user.name, alias=alias, website_from=forward_email.website_from, website_email=forward_email.website_email, ), render( "transactional/automatic-disable-alias.html", name=user.name, alias=alias, website_from=forward_email.website_from, website_email=forward_email.website_email, ), bounced_email=msg, )
def notify_manual_sub_end(): for manual_sub in ManualSubscription.all(): manual_sub: ManualSubscription need_reminder = False if arrow.now().shift(days=14) > manual_sub.end_at > arrow.now().shift( days=13): need_reminder = True elif arrow.now().shift(days=4) > manual_sub.end_at > arrow.now().shift( days=3): need_reminder = True user = manual_sub.user if user.lifetime: LOG.d("%s has a lifetime licence", user) continue paddle_sub: Subscription = user.get_subscription() if paddle_sub and not paddle_sub.cancelled: LOG.d("%s has an active Paddle subscription", user) continue if need_reminder: # user can have a (free) manual subscription but has taken a paid subscription via # Paddle, Coinbase or Apple since then if manual_sub.is_giveaway: if user.get_subscription(): LOG.d("%s has a active Paddle subscription", user) continue coinbase_subscription: CoinbaseSubscription = ( CoinbaseSubscription.get_by(user_id=user.id)) if coinbase_subscription and coinbase_subscription.is_active(): LOG.d("%s has a active Coinbase subscription", user) continue apple_sub: AppleSubscription = AppleSubscription.get_by( user_id=user.id) if apple_sub and apple_sub.is_valid(): LOG.d("%s has a active Apple subscription", user) continue LOG.d("Remind user %s that their manual sub is ending soon", user) send_email( user.email, f"Your subscription will end soon", render( "transactional/manual-subscription-end.txt", user=user, manual_sub=manual_sub, ), render( "transactional/manual-subscription-end.html", user=user, manual_sub=manual_sub, ), retries=3, ) extend_subscription_url = URL + "/dashboard/coinbase_checkout" for coinbase_subscription in CoinbaseSubscription.all(): need_reminder = False if (arrow.now().shift(days=14) > coinbase_subscription.end_at > arrow.now().shift(days=13)): need_reminder = True elif (arrow.now().shift(days=4) > coinbase_subscription.end_at > arrow.now().shift(days=3)): need_reminder = True if need_reminder: user = coinbase_subscription.user LOG.d( "Remind user %s that their coinbase subscription is ending soon", user) send_email( user.email, "Your SimpleLogin subscription will end soon", render( "transactional/coinbase/reminder-subscription.txt", coinbase_subscription=coinbase_subscription, extend_subscription_url=extend_subscription_url, ), render( "transactional/coinbase/reminder-subscription.html", coinbase_subscription=coinbase_subscription, extend_subscription_url=extend_subscription_url, ), retries=3, )
def stats(): """send admin stats everyday""" if not ADMIN_EMAIL: # nothing to do return # nb user q = User.query for ie in IGNORED_EMAILS: q = q.filter(~User.email.contains(ie)) nb_user = q.count() LOG.d("total number user %s", nb_user) # nb gen emails q = db.session.query(GenEmail, User).filter(GenEmail.user_id == User.id) for ie in IGNORED_EMAILS: q = q.filter(~User.email.contains(ie)) nb_gen_email = q.count() LOG.d("total number alias %s", nb_gen_email) # nb mails forwarded q = db.session.query(ForwardEmailLog, ForwardEmail, GenEmail, User).filter( ForwardEmailLog.forward_id == ForwardEmail.id, ForwardEmail.gen_email_id == GenEmail.id, GenEmail.user_id == User.id, ) for ie in IGNORED_EMAILS: q = q.filter(~User.email.contains(ie)) nb_forward = nb_block = nb_reply = 0 for fel, _, _, _ in q: if fel.is_reply: nb_reply += 1 elif fel.blocked: nb_block += 1 else: nb_forward += 1 LOG.d("nb forward %s, nb block %s, nb reply %s", nb_forward, nb_block, nb_reply) nb_premium = Subscription.query.count() nb_custom_domain = CustomDomain.query.count() nb_custom_domain_alias = GenEmail.query.filter( GenEmail.custom_domain_id.isnot(None)).count() nb_disabled_alias = GenEmail.query.filter( GenEmail.enabled == False).count() nb_app = Client.query.count() today = arrow.now().format() send_email( ADMIN_EMAIL, subject= f"SimpleLogin Stats for {today}, {nb_user} users, {nb_gen_email} aliases, {nb_forward} forwards", plaintext="", html=f""" Stats for {today} <br> nb_user: {nb_user} <br> nb_premium: {nb_premium} <br> nb_alias: {nb_gen_email} <br> nb_disabled_alias: {nb_disabled_alias} <br> nb_custom_domain: {nb_custom_domain} <br> nb_custom_domain_alias: {nb_custom_domain_alias} <br> nb_forward: {nb_forward} <br> nb_reply: {nb_reply} <br> nb_block: {nb_block} <br> nb_app: {nb_app} <br> """, )
def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: reply_email = rcpt_to.lower() # reply_email must end with EMAIL_DOMAIN if not reply_email.endswith(EMAIL_DOMAIN): LOG.warning(f"Reply email {reply_email} has wrong domain") return "550 wrong reply email" forward_email = ForwardEmail.get_by(reply_email=reply_email) if not forward_email: LOG.warning(f"No such forward-email with {reply_email} as reply-email") return "550 wrong reply email" alias: str = forward_email.gen_email.email alias_domain = alias[alias.find("@") + 1 :] # alias must end with one of the ALIAS_DOMAINS or custom-domain if not email_belongs_to_alias_domains(alias): if not CustomDomain.get_by(domain=alias_domain): return "550 alias unknown by SimpleLogin" gen_email = forward_email.gen_email user = gen_email.user mailbox_email = gen_email.mailbox_email() # bounce email initiated by Postfix # can happen in case emails cannot be delivered to user-email # in this case Postfix will try to send a bounce report to original sender, which is # the "reply email" if envelope.mail_from == "<>": LOG.error("Bounce when sending to alias %s, user %s", alias, gen_email.user) handle_bounce( alias, envelope, forward_email, gen_email, msg, smtp, user, mailbox_email ) return "550 ignored" # only mailbox can send email to the reply-email if envelope.mail_from.lower() != mailbox_email.lower(): LOG.warning( f"Reply email can only be used by user email. Actual mail_from: %s. msg from header: %s, User email %s. reply_email %s", envelope.mail_from, msg["From"], mailbox_email, reply_email, ) user = gen_email.user send_email( mailbox_email, f"Reply from your alias {alias} only works from your mailbox", render( "transactional/reply-must-use-personal-email.txt", name=user.name, alias=alias, sender=envelope.mail_from, mailbox_email=mailbox_email, ), render( "transactional/reply-must-use-personal-email.html", name=user.name, alias=alias, sender=envelope.mail_from, mailbox_email=mailbox_email, ), ) # Notify sender that they cannot send emails to this address send_email( envelope.mail_from, f"Your email ({envelope.mail_from}) is not allowed to send emails to {reply_email}", render( "transactional/send-from-alias-from-unknown-sender.txt", sender=envelope.mail_from, reply_email=reply_email, ), "", ) return "550 ignored" delete_header(msg, "DKIM-Signature") # the email comes from alias add_or_replace_header(msg, "From", alias) # some email providers like ProtonMail adds automatically the Reply-To field # make sure to delete it delete_header(msg, "Reply-To") # remove sender header if present as this could reveal user real email delete_header(msg, "Sender") add_or_replace_header(msg, "To", forward_email.website_email) # add List-Unsubscribe header unsubscribe_link = f"{URL}/dashboard/unsubscribe/{forward_email.gen_email_id}" add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>") add_or_replace_header(msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click") # Received-SPF is injected by postfix-policyd-spf-python can reveal user original email delete_header(msg, "Received-SPF") LOG.d( "send email from %s to %s, mail_options:%s,rcpt_options:%s", alias, forward_email.website_email, envelope.mail_options, envelope.rcpt_options, ) if alias_domain in ALIAS_DOMAINS: add_dkim_signature(msg, alias_domain) # add DKIM-Signature for custom-domain alias else: custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain) if custom_domain.dkim_verified: add_dkim_signature(msg, alias_domain) msg_raw = msg.as_string().encode() smtp.sendmail( alias, forward_email.website_email, msg_raw, envelope.mail_options, envelope.rcpt_options, ) ForwardEmailLog.create(forward_id=forward_email.id, is_reply=True) db.session.commit() return "250 Message accepted for delivery"
async def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str): """ return whether an email has been delivered and the smtp status ("250 Message accepted", "550 Non-existent email address", etc) """ reply_email = rcpt_to.lower().strip() # reply_email must end with EMAIL_DOMAIN if not reply_email.endswith(EMAIL_DOMAIN): LOG.warning(f"Reply email {reply_email} has wrong domain") return False, "550 SL E2" contact = Contact.get_by(reply_email=reply_email) if not contact: LOG.warning(f"No such forward-email with {reply_email} as reply-email") return False, "550 SL E4 Email not exist" alias = contact.alias address: str = contact.alias.email alias_domain = address[address.find("@") + 1:] # alias must end with one of the ALIAS_DOMAINS or custom-domain if not email_belongs_to_alias_domains(alias.email): if not CustomDomain.get_by(domain=alias_domain): return False, "550 SL E5" user = alias.user mail_from = envelope.mail_from.lower().strip() # bounce email initiated by Postfix # can happen in case emails cannot be delivered to user-email # in this case Postfix will try to send a bounce report to original sender, which is # the "reply email" if mail_from == "<>": LOG.warning( "Bounce when sending to alias %s from %s, user %s", alias, contact, user, ) handle_bounce(contact, alias, msg, user) return False, "550 SL E6" mailbox = Mailbox.get_by(email=mail_from, user_id=user.id) if not mailbox or mailbox not in alias.mailboxes: # only mailbox can send email to the reply-email handle_unknown_mailbox(envelope, msg, reply_email, user, alias) return False, "550 SL E7" if ENFORCE_SPF and mailbox.force_spf: ip = msg[_IP_HEADER] if not spf_pass(ip, envelope, mailbox, user, alias, contact.website_email, msg): # cannot use 4** here as sender will retry. 5** because that generates bounce report return True, "250 SL E11" email_log = EmailLog.create(contact_id=contact.id, is_reply=True, user_id=contact.user_id) # Spam check spam_status = "" is_spam = False # do not use user.max_spam_score here if SPAMASSASSIN_HOST: start = time.time() spam_score = await get_spam_score(msg) LOG.d( "%s -> %s - spam score %s in %s seconds", alias, contact, spam_score, time.time() - start, ) email_log.spam_score = spam_score if spam_score > MAX_REPLY_PHASE_SPAM_SCORE: is_spam = True spam_status = "Spam detected by SpamAssassin server" else: is_spam, spam_status = get_spam_info( msg, max_score=MAX_REPLY_PHASE_SPAM_SCORE) if is_spam: LOG.exception( "Reply phase - email sent from %s to %s detected as spam", alias, contact) email_log.is_spam = True email_log.spam_status = spam_status db.session.commit() handle_spam(contact, alias, msg, user, mailbox, email_log, is_reply=True) return False, "550 SL E15 Email detected as spam" delete_header(msg, _IP_HEADER) delete_header(msg, "DKIM-Signature") delete_header(msg, "Received") # make the email comes from alias from_header = alias.email # add alias name from alias if alias.name: LOG.d("Put alias name in from header") from_header = formataddr((alias.name, alias.email)) elif alias.custom_domain: LOG.d("Put domain default alias name in from header") # add alias name from domain if alias.custom_domain.name: from_header = formataddr((alias.custom_domain.name, alias.email)) add_or_replace_header(msg, "From", from_header) # some email providers like ProtonMail adds automatically the Reply-To field # make sure to delete it delete_header(msg, "Reply-To") # remove sender header if present as this could reveal user real email delete_header(msg, "Sender") delete_header(msg, "X-Sender") replace_header_when_reply(msg, alias, "To") replace_header_when_reply(msg, alias, "Cc") add_or_replace_header( msg, _MESSAGE_ID, make_msgid(str(email_log.id), get_email_domain_part(alias.email)), ) add_or_replace_header(msg, _EMAIL_LOG_ID_HEADER, str(email_log.id)) add_or_replace_header(msg, _DIRECTION, "Reply") # Received-SPF is injected by postfix-policyd-spf-python can reveal user original email delete_header(msg, "Received-SPF") LOG.d( "send email from %s to %s, mail_options:%s,rcpt_options:%s", alias.email, contact.website_email, envelope.mail_options, envelope.rcpt_options, ) # replace "*****@*****.**" by the contact email in the email body # as this is usually included when replying if user.replace_reverse_alias: if msg.is_multipart(): for part in msg.walk(): if part.get_content_maintype() != "text": continue part = replace_str_in_msg(part, reply_email, contact.website_email) else: msg = replace_str_in_msg(msg, reply_email, contact.website_email) if alias_domain in ALIAS_DOMAINS: add_dkim_signature(msg, alias_domain) # add DKIM-Signature for custom-domain alias else: custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain) if custom_domain.dkim_verified: add_dkim_signature(msg, alias_domain) # create PGP email if needed if contact.pgp_finger_print and user.is_premium(): LOG.d("Encrypt message for contact %s", contact) try: msg = prepare_pgp_message(msg, contact.pgp_finger_print) except PGPException: LOG.exception("Cannot encrypt message %s -> %s. %s %s", alias, contact, mailbox, user) # to not save the email_log db.session.rollback() # return 421 so the client can retry later return False, "421 SL E13 Retry later" try: smtp.sendmail( alias.email, contact.website_email, msg.as_bytes(), envelope.mail_options, envelope.rcpt_options, ) except Exception: # to not save the email_log db.session.rollback() LOG.exception("Cannot send email from %s to %s", alias, contact) send_email( mailbox.email, f"Email cannot be sent to {contact.email} from {alias.email}", render( "transactional/reply-error.txt", user=user, alias=alias, contact=contact, contact_domain=get_email_domain_part(contact.email), ), render( "transactional/reply-error.html", user=user, alias=alias, contact=contact, contact_domain=get_email_domain_part(contact.email), ), ) db.session.commit() return True, "250 Message accepted for delivery"
def mailbox_route(): mailboxes = (Mailbox.query.filter_by(user_id=current_user.id).order_by( Mailbox.created_at.desc()).all()) new_mailbox_form = NewMailboxForm() if request.method == "POST": if request.form.get("form-name") == "delete": mailbox_id = request.form.get("mailbox-id") mailbox = Mailbox.get(mailbox_id) if not mailbox or mailbox.user_id != current_user.id: flash("Unknown error. Refresh the page", "warning") return redirect(url_for("dashboard.mailbox_route")) if mailbox.id == current_user.default_mailbox_id: flash("You cannot delete default mailbox", "error") return redirect(url_for("dashboard.mailbox_route")) email = mailbox.email Mailbox.delete(mailbox_id) db.session.commit() flash(f"Mailbox {email} has been deleted", "success") return redirect(url_for("dashboard.mailbox_route")) if request.form.get("form-name") == "set-default": mailbox_id = request.form.get("mailbox-id") mailbox = Mailbox.get(mailbox_id) if not mailbox or mailbox.user_id != current_user.id: flash("Unknown error. Refresh the page", "warning") return redirect(url_for("dashboard.mailbox_route")) if mailbox.id == current_user.default_mailbox_id: flash("This mailbox is already default one", "error") return redirect(url_for("dashboard.mailbox_route")) if not mailbox.verified: flash("Cannot set unverified mailbox as default", "error") return redirect(url_for("dashboard.mailbox_route")) current_user.default_mailbox_id = mailbox.id db.session.commit() flash(f"Mailbox {mailbox.email} is set as Default Mailbox", "success") return redirect(url_for("dashboard.mailbox_route")) elif request.form.get("form-name") == "create": if not current_user.is_premium(): flash("Only premium plan can add additional mailbox", "warning") return redirect(url_for("dashboard.mailbox_route")) if new_mailbox_form.validate(): mailbox_email = new_mailbox_form.email.data.lower() if mailbox_already_used(mailbox_email, current_user): flash(f"{mailbox_email} already used", "error") elif not email_domain_can_be_used_as_mailbox(mailbox_email): flash(f"You cannot use {mailbox_email}.", "error") else: new_mailbox = Mailbox.create(email=mailbox_email, user_id=current_user.id) db.session.commit() s = Signer(MAILBOX_SECRET) mailbox_id_signed = s.sign(str(new_mailbox.id)).decode() verification_url = (URL + "/dashboard/mailbox_verify" + f"?mailbox_id={mailbox_id_signed}") send_email( mailbox_email, f"Please confirm your email {mailbox_email}", render( "transactional/verify-mailbox.txt", user=current_user, link=verification_url, mailbox_email=mailbox_email, ), render( "transactional/verify-mailbox.html", user=current_user, link=verification_url, mailbox_email=mailbox_email, ), ) flash( f"You are going to receive an email to confirm {mailbox_email}.", "success", ) return redirect( url_for("dashboard.mailbox_detail_route", mailbox_id=new_mailbox.id)) return render_template( "dashboard/mailbox.html", mailboxes=mailboxes, new_mailbox_form=new_mailbox_form, EMAIL_DOMAIN=EMAIL_DOMAIN, ALIAS_DOMAINS=ALIAS_DOMAINS, )
def sanity_check(): """ #TODO: investigate why DNS sometimes not working Different sanity checks - detect if there's mailbox that's using a invalid domain """ mailbox_ids = (db.session.query(Mailbox.id).filter( Mailbox.verified == True, Mailbox.disabled == False).all()) mailbox_ids = [e[0] for e in mailbox_ids] # iterate over id instead of mailbox directly # as a mailbox can be deleted during the sleep time for mailbox_id in mailbox_ids: mailbox = Mailbox.get(mailbox_id) # a mailbox has been deleted if not mailbox: continue # hack to not query DNS too often sleep(1) if not email_domain_can_be_used_as_mailbox(mailbox.email): mailbox.nb_failed_checks += 1 nb_email_log = nb_email_log_for_mailbox(mailbox) log_func = LOG.warning # send a warning if mailbox.nb_failed_checks == 5: if mailbox.user.email != mailbox.email: send_email( mailbox.user.email, f"Mailbox {mailbox.email} is disabled", render("transactional/disable-mailbox-warning.txt", mailbox=mailbox), render( "transactional/disable-mailbox-warning.html", mailbox=mailbox, ), ) # alert if too much fail and nb_email_log > 100 if mailbox.nb_failed_checks > 10 and nb_email_log > 100: log_func = LOG.exception mailbox.disabled = True if mailbox.user.email != mailbox.email: send_email( mailbox.user.email, f"Mailbox {mailbox.email} is disabled", render("transactional/disable-mailbox.txt", mailbox=mailbox), render("transactional/disable-mailbox.html", mailbox=mailbox), ) log_func( "issue with mailbox %s domain. #alias %s, nb email log %s", mailbox, mailbox.nb_alias(), nb_email_log, ) else: # reset nb check mailbox.nb_failed_checks = 0 db.session.commit() for user in User.filter_by(activated=True).all(): if user.email.lower().strip().replace(" ", "") != user.email: LOG.exception("%s does not have sanitized email", user) for alias in Alias.query.all(): if alias.email.lower().strip().replace(" ", "") != alias.email: LOG.exception("Alias %s email not sanitized", alias) for contact in Contact.query.all(): if contact.reply_email.lower().strip().replace( " ", "") != contact.reply_email: LOG.exception("Contact %s reply-email not sanitized", contact) for mailbox in Mailbox.query.all(): if mailbox.email.lower().strip().replace(" ", "") != mailbox.email: LOG.exception("Mailbox %s address not sanitized", mailbox) LOG.d("Finish sanity check")
def stats(): """send admin stats everyday""" if not ADMIN_EMAIL: # nothing to do return # todo: remove metrics1 compute_metrics() stats_today = compute_metric2() stats_yesterday = (Metric2.query.filter( Metric2.date < stats_today.date).order_by(Metric2.date.desc()).first()) nb_user_increase = increase_percent(stats_yesterday.nb_user, stats_today.nb_user) nb_alias_increase = increase_percent(stats_yesterday.nb_alias, stats_today.nb_alias) nb_forward_increase = increase_percent(stats_yesterday.nb_forward, stats_today.nb_forward) today = arrow.now().format() html = f""" Stats for {today} <br> nb_user: {stats_today.nb_user} - {increase_percent(stats_yesterday.nb_user, stats_today.nb_user)} <br> nb_premium: {stats_today.nb_premium} - {increase_percent(stats_yesterday.nb_premium, stats_today.nb_premium)} <br> nb_cancelled_premium: {stats_today.nb_cancelled_premium} - {increase_percent(stats_yesterday.nb_cancelled_premium, stats_today.nb_cancelled_premium)} <br> nb_apple_premium: {stats_today.nb_apple_premium} - {increase_percent(stats_yesterday.nb_apple_premium, stats_today.nb_apple_premium)} <br> nb_manual_premium: {stats_today.nb_manual_premium} - {increase_percent(stats_yesterday.nb_manual_premium, stats_today.nb_manual_premium)} <br> nb_coinbase_premium: {stats_today.nb_coinbase_premium} - {increase_percent(stats_yesterday.nb_coinbase_premium, stats_today.nb_coinbase_premium)} <br> nb_alias: {stats_today.nb_alias} - {increase_percent(stats_yesterday.nb_alias, stats_today.nb_alias)} <br> nb_forward: {stats_today.nb_forward} - {increase_percent(stats_yesterday.nb_forward, stats_today.nb_forward)} <br> nb_reply: {stats_today.nb_reply} - {increase_percent(stats_yesterday.nb_reply, stats_today.nb_reply)} <br> nb_block: {stats_today.nb_block} - {increase_percent(stats_yesterday.nb_block, stats_today.nb_block)} <br> nb_bounced: {stats_today.nb_bounced} - {increase_percent(stats_yesterday.nb_bounced, stats_today.nb_bounced)} <br> nb_spam: {stats_today.nb_spam} - {increase_percent(stats_yesterday.nb_spam, stats_today.nb_spam)} <br> nb_custom_domain: {stats_today.nb_verified_custom_domain} - {increase_percent(stats_yesterday.nb_verified_custom_domain, stats_today.nb_verified_custom_domain)} <br> nb_app: {stats_today.nb_app} - {increase_percent(stats_yesterday.nb_app, stats_today.nb_app)} <br> nb_referred_user: {stats_today.nb_referred_user} - {increase_percent(stats_yesterday.nb_referred_user, stats_today.nb_referred_user)} <br> nb_referred_user_upgrade: {stats_today.nb_referred_user_paid} - {increase_percent(stats_yesterday.nb_referred_user_paid, stats_today.nb_referred_user_paid)} <br> """ html += f"""<br> Bounce report: <br> """ for email, bounces in bounce_report(): html += f"{email}: {bounces} <br>" html += f"""<br><br> Alias creation report: <br> """ for email, nb_alias, date in alias_creation_report(): html += f"{email}, {date}: {nb_alias} <br>" LOG.d("report email: %s", html) send_email( ADMIN_EMAIL, subject= f"SimpleLogin Stats for {today}, {nb_user_increase} users, {nb_alias_increase} aliases, {nb_forward_increase} forwards", plaintext="", html=html, )
def coupon_route(): if current_user.lifetime: flash("You already have a lifetime licence", "warning") return redirect(url_for("dashboard.index")) # handle case user already has an active subscription via another channel (Paddle, Apple, etc) if current_user._lifetime_or_active_subscription(): manual_sub: ManualSubscription = ManualSubscription.get_by( user_id=current_user.id) # user has an non-manual subscription if not manual_sub or not manual_sub.is_active(): flash("You already have another subscription.", "warning") return redirect(url_for("dashboard.index")) coupon_form = CouponForm() if coupon_form.validate_on_submit(): code = coupon_form.code.data coupon: Coupon = Coupon.get_by(code=code) if coupon and not coupon.used: coupon.used_by_user_id = current_user.id coupon.used = True db.session.commit() manual_sub: ManualSubscription = ManualSubscription.get_by( user_id=current_user.id) if manual_sub: # renew existing subscription if manual_sub.end_at > arrow.now(): manual_sub.end_at = manual_sub.end_at.shift( years=coupon.nb_year) else: manual_sub.end_at = arrow.now().shift(years=coupon.nb_year, days=1) db.session.commit() flash( f"Your current subscription is extended to {manual_sub.end_at.humanize()}", "success", ) else: ManualSubscription.create( user_id=current_user.id, end_at=arrow.now().shift(years=coupon.nb_year, days=1), comment="using coupon code", is_giveaway=False, commit=True, ) flash( f"Your account has been upgraded to Premium, thanks for your support!", "success", ) # notify admin send_email( ADMIN_EMAIL, subject=f"User {current_user} applies the coupon", plaintext="", html="", ) return redirect(url_for("dashboard.index")) else: flash(f"Code *{code}* expired or invalid", "warning") return render_template("dashboard/coupon.html", coupon_form=coupon_form)
def mailbox_detail_route(mailbox_id): mailbox = Mailbox.get(mailbox_id) if not mailbox or mailbox.user_id != current_user.id: flash("You cannot see this page", "warning") return redirect(url_for("dashboard.index")) change_email_form = ChangeEmailForm() if mailbox.new_email: pending_email = mailbox.new_email else: pending_email = None if request.method == "POST": if ( request.form.get("form-name") == "update-email" and change_email_form.validate_on_submit() ): new_email = change_email_form.email.data if new_email != mailbox.email and not pending_email: # check if this email is not already used if ( mailbox_already_used(new_email, current_user) or Alias.get_by(email=new_email) or DeletedAlias.get_by(email=new_email) ): flash(f"Email {new_email} already used", "error") elif not email_domain_can_be_used_as_mailbox(new_email): flash("You cannot use this email address as your mailbox", "error") else: mailbox.new_email = new_email db.session.commit() s = Signer(MAILBOX_SECRET) mailbox_id_signed = s.sign(str(mailbox.id)).decode() verification_url = ( URL + "/dashboard/mailbox/confirm_change" + f"?mailbox_id={mailbox_id_signed}" ) try: send_email( new_email, f"Confirm mailbox change on SimpleLogin", render( "transactional/verify-mailbox-change.txt", user=current_user, link=verification_url, mailbox_email=mailbox.email, mailbox_new_email=new_email, ), render( "transactional/verify-mailbox-change.html", user=current_user, link=verification_url, mailbox_email=mailbox.email, mailbox_new_email=new_email, ), ) except SMTPRecipientsRefused: flash( f"Incorrect mailbox, please recheck {mailbox.email}", "error", ) else: flash( f"You are going to receive an email to confirm {new_email}.", "success", ) return redirect( url_for("dashboard.mailbox_detail_route", mailbox_id=mailbox_id) ) elif request.form.get("form-name") == "force-spf": if not ENFORCE_SPF: flash("SPF enforcement globally not enabled", "error") return redirect(url_for("dashboard.index")) mailbox.force_spf = ( True if request.form.get("spf-status") == "on" else False ) db.session.commit() flash( "SPF enforcement was " + "enabled" if request.form.get("spf-status") else "disabled" + " succesfully", "success", ) return redirect( url_for("dashboard.mailbox_detail_route", mailbox_id=mailbox_id) ) elif request.form.get("form-name") == "pgp": if request.form.get("action") == "save": if not current_user.is_premium(): flash("Only premium plan can add PGP Key", "warning") return redirect( url_for("dashboard.mailbox_detail_route", mailbox_id=mailbox_id) ) mailbox.pgp_public_key = request.form.get("pgp") try: mailbox.pgp_finger_print = load_public_key(mailbox.pgp_public_key) except PGPException: flash("Cannot add the public key, please verify it", "error") else: db.session.commit() flash("Your PGP public key is saved successfully", "success") return redirect( url_for("dashboard.mailbox_detail_route", mailbox_id=mailbox_id) ) elif request.form.get("action") == "remove": # Free user can decide to remove their added PGP key mailbox.pgp_public_key = None mailbox.pgp_finger_print = None db.session.commit() flash("Your PGP public key is removed successfully", "success") return redirect( url_for("dashboard.mailbox_detail_route", mailbox_id=mailbox_id) ) spf_available = ENFORCE_SPF return render_template("dashboard/mailbox_detail.html", **locals())
user_id = job.payload.get("user_id") user = User.get(user_id) if not user: LOG.i("No user found for %s", user_id) continue user_email = user.email LOG.w("Delete user %s", user) User.delete(user.id) Session.commit() send_email( user_email, "Your SimpleLogin account has been deleted", render("transactional/account-delete.txt"), render("transactional/account-delete.html"), retries=3, ) elif job.name == JOB_DELETE_MAILBOX: mailbox_id = job.payload.get("mailbox_id") mailbox = Mailbox.get(mailbox_id) if not mailbox: continue mailbox_email = mailbox.email user = mailbox.user Mailbox.delete(mailbox_id) Session.commit() LOG.d("Mailbox %s %s deleted", mailbox_id, mailbox_email)