def test_rate_limited_forward_phase_for_alias(flask_client): user = User.create(email="[email protected]", password="******", name="Test User", activated=True) db.session.commit() # no rate limiting for a new alias alias = Alias.create_new_random(user) db.session.commit() assert not rate_limited_for_alias(alias) # rate limit when there's a previous activity on alias contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", ) db.session.commit() for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_ALIAS + 1): EmailLog.create(user_id=user.id, contact_id=contact.id) db.session.commit() assert rate_limited_for_alias(alias)
def test_alias_activities(flask_client): user = User.create(email="[email protected]", password="******", name="Test User", activated=True) Session.commit() # create api_key api_key = ApiKey.create(user.id, "for test") Session.commit() alias = Alias.create_new_random(user) Session.commit() # create some alias log contact = Contact.create( website_email="*****@*****.**", reply_email="[email protected]", alias_id=alias.id, user_id=alias.user_id, ) Session.commit() for _ in range(int(PAGE_LIMIT / 2)): EmailLog.create( contact_id=contact.id, is_reply=True, user_id=contact.user_id, alias_id=contact.alias_id, ) for _ in range(int(PAGE_LIMIT / 2) + 2): EmailLog.create( contact_id=contact.id, blocked=True, user_id=contact.user_id, alias_id=contact.alias_id, ) r = flask_client.get( url_for("api.get_alias_activities", alias_id=alias.id, page_id=0), headers={"Authentication": api_key.code}, ) assert r.status_code == 200 assert len(r.json["activities"]) == PAGE_LIMIT for ac in r.json["activities"]: assert ac["from"] assert ac["to"] assert ac["timestamp"] assert ac["action"] assert ac["reverse_alias"] assert ac["reverse_alias_address"] # second page, should return 1 or 2 results only r = flask_client.get( url_for("api.get_alias_activities", alias_id=alias.id, page_id=1), headers={"Authentication": api_key.code}, ) assert len(r.json["activities"]) < 3
def test_greylisting_needed_forward_phase_for_mailbox(flask_client): user = User.create( email="[email protected]", password="******", name="Test User", activated=True ) db.session.commit() alias = Alias.create_new_random(user) db.session.commit() contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", ) db.session.commit() for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_MAILBOX + 1): EmailLog.create(user_id=user.id, contact_id=contact.id) db.session.commit() EmailLog.create(user_id=user.id, contact_id=contact.id) # Create another alias with the same mailbox # will be greylisted as there's a previous activity on mailbox alias2 = Alias.create_new_random(user) db.session.commit() assert greylisting_needed_for_mailbox(alias2)
def test_rate_limited_reply_phase(flask_client): # no rate limiting when reply_email does not exist assert not rate_limited_reply_phase("*****@*****.**") user = User.create(email="[email protected]", password="******", name="Test User", activated=True) db.session.commit() alias = Alias.create_new_random(user) db.session.commit() contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", ) db.session.commit() for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_ALIAS + 1): EmailLog.create(user_id=user.id, contact_id=contact.id) db.session.commit() assert rate_limited_reply_phase("*****@*****.**")
def test_should_disable(flask_client): user = User.create( email="[email protected]", password="******", name="Test User", activated=True, include_sender_in_reverse_alias=True, ) alias = Alias.create_new_random(user) db.session.commit() assert not should_disable(alias) # create a lot of bounce on this alias contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", commit=True, ) for _ in range(20): EmailLog.create(user_id=user.id, contact_id=contact.id, commit=True, bounced=True) assert should_disable(alias) # should not affect another alias alias2 = Alias.create_new_random(user) db.session.commit() assert not should_disable(alias2)
def test_should_disable_bounces_every_day(flask_client): """if an alias has bounces every day at least 9 days in the last 10 days, disable alias""" user = login(flask_client) alias = Alias.create_new_random(user) Session.commit() assert not should_disable(alias) # create a lot of bounce on this alias contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", commit=True, ) for i in range(9): EmailLog.create( user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id, commit=True, bounced=True, created_at=arrow.now().shift(days=-i), ) assert should_disable(alias)
def test_should_disable_bounces_account(flask_client): """if an account has more than 10 bounces every day for at least 5 days in the last 10 days, disable alias""" user = login(flask_client) alias = Alias.create_new_random(user) Session.commit() # create a lot of bounces on alias contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", commit=True, ) for day in range(6): for _ in range(10): EmailLog.create( user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id, commit=True, bounced=True, created_at=arrow.now().shift(days=-day), ) alias2 = Alias.create_new_random(user) assert should_disable(alias2)
def test_should_disable_bounce_consecutive_days(flask_client): user = login(flask_client) alias = Alias.create_new_random(user) Session.commit() contact = Contact.create( user_id=user.id, alias_id=alias.id, website_email="*****@*****.**", reply_email="*****@*****.**", commit=True, ) # create 6 bounce on this alias in the last 24h: alias is not disabled for _ in range(6): EmailLog.create( user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id, commit=True, bounced=True, ) assert not should_disable(alias) # create 2 bounces in the last 7 days: alias should be disabled for _ in range(2): EmailLog.create( user_id=user.id, contact_id=contact.id, alias_id=contact.alias_id, commit=True, bounced=True, created_at=arrow.now().shift(days=-3), ) assert should_disable(alias)
def send_email(email_log_id): email = EmailLog[email_log_id] rv = mailext.send_email(email.to, email.subject, email.body) email_id = rv['id'] EmailLog.set_by_id( email_log_id, { 'email_id': email_id, 'status': EmailLog.SEND_MAILGUN, 'updated_at': datetime.datetime.now() })
def compute_metric2() -> Metric2: now = arrow.now() _24h_ago = now.shift(days=-1) nb_referred_user_paid = 0 for user in User.filter(User.referral_id.isnot(None)): if user.is_paid(): nb_referred_user_paid += 1 return Metric2.create( date=now, # user stats nb_user=User.count(), nb_activated_user=User.filter_by(activated=True).count(), # subscription stats nb_premium=Subscription.filter( Subscription.cancelled.is_(False)).count(), nb_cancelled_premium=Subscription.filter( Subscription.cancelled.is_(True)).count(), # todo: filter by expires_date > now nb_apple_premium=AppleSubscription.count(), nb_manual_premium=ManualSubscription.filter( ManualSubscription.end_at > now, ManualSubscription.is_giveaway.is_(False), ).count(), nb_coinbase_premium=CoinbaseSubscription.filter( CoinbaseSubscription.end_at > now).count(), # referral stats nb_referred_user=User.filter(User.referral_id.isnot(None)).count(), nb_referred_user_paid=nb_referred_user_paid, nb_alias=Alias.count(), # email log stats nb_forward_last_24h=EmailLog.filter( EmailLog.created_at > _24h_ago).filter_by(bounced=False, is_spam=False, is_reply=False, blocked=False).count(), nb_bounced_last_24h=EmailLog.filter( EmailLog.created_at > _24h_ago).filter_by(bounced=True).count(), nb_total_bounced_last_24h=Bounce.filter( Bounce.created_at > _24h_ago).count(), nb_reply_last_24h=EmailLog.filter( EmailLog.created_at > _24h_ago).filter_by(is_reply=True).count(), nb_block_last_24h=EmailLog.filter( EmailLog.created_at > _24h_ago).filter_by(blocked=True).count(), # other stats nb_verified_custom_domain=CustomDomain.filter_by( verified=True).count(), nb_subdomain=CustomDomain.filter_by(is_sl_subdomain=True).count(), nb_directory=Directory.count(), nb_deleted_directory=DeletedDirectory.count(), nb_deleted_subdomain=DeletedSubdomain.count(), nb_app=Client.count(), commit=True, )
def schedule_emails(): now = datetime.datetime.now() limit_time = now + datetime.timedelta(minutes=10) email_objs = EmailLog.select().where(EmailLog.status == EmailLog.SCHEDULED, EmailLog.send_at <= limit_time) for i in email_objs: EmailLog.set_by_id(i.id, { 'status': EmailLog.IN_QUEUE, 'updated_at': datetime.datetime.now() }) send_email.apply_async(args=(i.id, ), eta=i.send_at)
def test_alias_contacts(flask_client): user = User.create( email="[email protected]", password="******", name="Test User", activated=True ) db.session.commit() # create api_key api_key = ApiKey.create(user.id, "for test") db.session.commit() alias = Alias.create_new_random(user) db.session.commit() # create some alias log for i in range(PAGE_LIMIT + 1): contact = Contact.create( website_email=f"marketing-{i}@example.com", reply_email=f"reply-{i}@a.b", alias_id=alias.id, user_id=alias.user_id, ) db.session.commit() EmailLog.create( contact_id=contact.id, is_reply=True, user_id=contact.user_id, alias_id=contact.alias_id, ) db.session.commit() r = flask_client.get( url_for("api.get_alias_contacts_route", alias_id=alias.id, page_id=0), headers={"Authentication": api_key.code}, ) assert r.status_code == 200 assert len(r.json["contacts"]) == PAGE_LIMIT for ac in r.json["contacts"]: assert ac["creation_date"] assert ac["creation_timestamp"] assert ac["last_email_sent_date"] assert ac["last_email_sent_timestamp"] assert ac["contact"] assert ac["reverse_alias"] assert ac["reverse_alias_address"] # second page, should return 1 result only r = flask_client.get( url_for("api.get_alias_contacts_route", alias_id=alias.id, page_id=1), headers={"Authentication": api_key.code}, ) assert len(r.json["contacts"]) == 1
def test_delay_email(mocker, email_logs, faker): from app.tasks import send_email send_mock = mocker.patch.object( send_email, 'apply_async', ) schedule_emails() now = datetime.datetime.now() query = EmailLog.select().where( EmailLog.send_at <= now + datetime.timedelta(minutes=10), EmailLog.status != EmailLog.IN_QUEUE) assert query.count() == 0 query = EmailLog.select().where(EmailLog.status == EmailLog.IN_QUEUE, ) assert query.count() == send_mock.call_count
def post(self): data = request.get_json() event_data = data['event-data'] if event_data['event'] != 'opened' or not mailext.verify( **data['signature']): return Response(status=406) email_id = event_data['id'] email = EmailLog.select().where(EmailLog.email_id == email_id).first() if email is None or email.status != EmailLog.SEND_SUCCESSFULLY: return Response(status=406) EmailLog.set_by_id(email.id, { 'status': EmailLog.OPENED, 'updated_at': datetime.datetime.now() }) return Response(status=200)
def send_confirm_email(self, user): tracking_code = email_service.confirm_email(user) log = EmailLog(user_id=user.id, type="confirm_email", tracking_code=tracking_code) db.session.add(log) db.session.commit()
def test_email_track_open_successfully(client, mocker, faker, email_logs_called_mailgun): from app.api.views import mailext verify_mock = mocker.patch.object(mailext, 'verify', return_val=True) email = EmailLog.select().first() EmailLog.set_by_id(email.id, { 'status': EmailLog.SEND_SUCCESSFULLY, }) resp = client.post('/email/open-track', json={ 'event-data': { 'event': 'opened', 'id': email.email_id, }, 'signature': {} }) assert resp.status_code == 200 assert EmailLog[email.id].status == EmailLog.OPENED assert verify_mock.called
async def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]: """return whether an email has been delivered and the smtp status ("250 Message accepted", "550 Non-existent email address", etc) """ address = rcpt_to.lower().strip() # alias@SL alias = Alias.get_by(email=address) if not alias: LOG.d("alias %s not exist. Try to see if it can be created on the fly", address) alias = try_auto_create(address) if not alias: LOG.d("alias %s cannot be created on-the-fly, return 550", address) return [(False, "550 SL E3 Email not exist")] mail_from = envelope.mail_from.lower().strip() for mb in alias.mailboxes: # email send from a mailbox to alias if mb.email.lower().strip() == mail_from: LOG.exception("cycle email sent from %s to %s", mb, alias) handle_email_sent_to_ourself(alias, mb, msg, alias.user) return [(True, "250 Message accepted for delivery")] contact = get_or_create_contact(msg["From"], envelope.mail_from, alias) email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id) db.session.commit() if not alias.enabled: LOG.d("%s is disabled, do not forward", alias) email_log.blocked = True db.session.commit() # do not return 5** to allow user to receive emails later when alias is enabled return [(True, "250 Message accepted for delivery")] user = alias.user ret = [] mailboxes = alias.mailboxes # no need to create a copy of message if len(mailboxes) == 1: mailbox = mailboxes[0] ret.append(await forward_email_to_mailbox(alias, msg, email_log, contact, envelope, smtp, mailbox, user)) # create a copy of message for each forward else: for mailbox in mailboxes: ret.append(await forward_email_to_mailbox(alias, copy(msg), email_log, contact, envelope, smtp, mailbox, user)) return ret
def test_send_email(mocker, email_logs, faker): from app.tasks import mailext return_val = {'id': faker.isbn13()} send_mock = mocker.patch.object(mailext, 'send_email', return_value=return_val) email = EmailLog.select().first() send_email(email.id) assert send_mock.called == 1 assert EmailLog[email.id].status == EmailLog.SEND_MAILGUN assert EmailLog[email.id].email_id == return_val['id']
def test_alias_contacts(flask_client): user = login(flask_client) alias = Alias.create_new_random(user) Session.commit() # create some alias log for i in range(PAGE_LIMIT + 1): contact = Contact.create( website_email=f"marketing-{i}@example.com", reply_email=f"reply-{i}@a.b", alias_id=alias.id, user_id=alias.user_id, ) Session.commit() EmailLog.create( contact_id=contact.id, is_reply=True, user_id=contact.user_id, alias_id=contact.alias_id, ) Session.commit() r = flask_client.get(f"/api/aliases/{alias.id}/contacts?page_id=0") assert r.status_code == 200 assert len(r.json["contacts"]) == PAGE_LIMIT for ac in r.json["contacts"]: assert ac["creation_date"] assert ac["creation_timestamp"] assert ac["last_email_sent_date"] assert ac["last_email_sent_timestamp"] assert ac["contact"] assert ac["reverse_alias"] assert ac["reverse_alias_address"] assert "block_forward" in ac # second page, should return 1 result only r = flask_client.get(f"/api/aliases/{alias.id}/contacts?page_id=1") assert len(r.json["contacts"]) == 1
def post(self): data = request.get_json() event_data = data['event-data'] if event_data['event'] not in ( 'delivered', 'failed', ) or not mailext.verify(**data['signature']): return Response(status=406) email_id = event_data['id'] email = EmailLog.select().where(EmailLog.email_id == email_id).first() if email is None or email.status != EmailLog.SEND_MAILGUN: return Response(status=406) if event_data['event'] == 'failed': EmailLog.set_by_id(email.id, {'status': EmailLog.SEND_FAILED}) else: EmailLog.set_by_id( email.id, { 'status': EmailLog.SEND_SUCCESSFULLY, 'updated_at': datetime.datetime.now() }) return Response(status=200)
def forgot_password(): request_data = request.get_json() email = request_data['email'] user = User.query.filter( func.lower(User.email) == email.lower()).first_or_404() tracking_code = email_service.reset_email(user) log = EmailLog(user_id=user.id, type="reset_email", tracking_code=tracking_code) db.session.add(log) db.session.commit() return ''
def approval_request(): request_data = request.get_json() user_id = request_data['user_id'] resource_id = request_data['resource_id'] resource = ThrivResource.query.filter_by(id=resource_id).first_or_404() # Get admins for the user's institution user = User.query.filter_by(id=user_id).first_or_404() admins = User.query.filter_by(role="Admin", institution_id=user.institution_id).all() # Send approval request email to each admin for the institution for admin in admins: tracking_code = email_service.approval_request_email(user, admin, resource) log = EmailLog(user_id=admin.id, type="approval_request", tracking_code=tracking_code) db.session.add(log) db.session.commit() # Send confirmation email to user confirm_tracking_code = email_service.approval_request_confirm_email(user, resource) confirm_log = EmailLog(user_id=user.id, type="approval_request_confirm", tracking_code=confirm_tracking_code) db.session.add(confirm_log) db.session.commit() return ''
def test_schedule_email_by_countdown(client, faker): data = { 'to': faker.email(), 'countdown': { 'hours': faker.pyint(), 'minutes': faker.pyint(), 'seconds': faker.pyint(), }, 'body': faker.text(), 'subject': faker.sentence() } resp = client.post('/email/schedule', json=data) assert resp.status_code == 200 email = EmailLog.select().first() assert email.created_at + datetime.timedelta( **data['countdown']) == email.send_at
def test_email_track_send_failed(client, faker, mocker, email_logs_called_mailgun): from app.api.views import mailext verify_mock = mocker.patch.object(mailext, 'verify', return_val=True) email = EmailLog.select().first() resp = client.post('/email/send-track', json={ 'event-data': { 'event': 'failed', 'id': email.email_id, }, 'signature': {} }) assert resp.status_code == 200 assert EmailLog[email.id].status == EmailLog.SEND_FAILED assert verify_mock.called
def consult_request(): request_data = request.get_json() user_id = request_data['user_id'] user = User.query.filter_by(id=user_id).first_or_404() admins = User.query.filter_by(role="Admin", institution_id=user.institution_id).all() # Send consult request email to each admin for the institution for admin in admins: tracking_code = email_service.consult_email(user, admin, request_data) log = EmailLog(user_id=admin.id, type="consult_request", tracking_code=tracking_code) db.session.add(log) db.session.commit() return ''
def test_schedule_email_by_eta(client, mocker, faker): from app.tasks import send_email mocker.patch.object( send_email, 'apply_async', ) data = { 'to': faker.email(), 'eta': str(faker.future_datetime()), 'body': faker.text(), 'subject': faker.sentence() } resp = client.post('/email/schedule', json=data) assert resp.status_code == 200 email = EmailLog.select().first() assert email.to == data['to'] assert str(email.send_at) == data['eta'] assert email.body == data['body']
def fill_up_email_log_alias(): """Fill up email_log.alias_id column""" # split all emails logs into 1000-size trunks nb_email_log = EmailLog.count() LOG.d("total trunks %s", nb_email_log // 1000 + 2) for trunk in reversed(range(1, nb_email_log // 1000 + 2)): nb_update = 0 for email_log, contact in (Session.query( EmailLog, Contact).filter(EmailLog.contact_id == Contact.id).filter( EmailLog.id <= trunk * 1000).filter( EmailLog.id > (trunk - 1) * 1000).filter( EmailLog.alias_id.is_(None))): email_log.alias_id = contact.alias_id nb_update += 1 LOG.d("finish trunk %s, update %s email logs", trunk, nb_update) Session.commit()
def post(self): data = request.get_json() now = datetime.datetime.now() if data.get('eta') is not None: send_at = datetime.datetime.strptime(data['eta'], '%Y-%m-%d %H:%M:%S') if send_at < now: raise ScheduleTimeException( 'schedule time:{} error'.format(send_at)) else: countdown = data['countdown'] send_at = datetime.datetime.now() + datetime.timedelta(**countdown) email = EmailLog.create(subject=data['subject'], body=data['body'], to=data['to'], send_at=send_at) if send_at - now <= datetime.timedelta(minutes=15): send_email.apply_async(args=(email.id, ), eta=send_at) return jsonify()
def delete_logs(): """delete everything that are considered logs""" delete_refused_emails() delete_old_monitoring() for t in TransactionalEmail.filter( TransactionalEmail.created_at < arrow.now().shift(days=-7)): TransactionalEmail.delete(t.id) for b in Bounce.filter(Bounce.created_at < arrow.now().shift(days=-7)): Bounce.delete(b.id) Session.commit() LOG.d("Delete EmailLog older than 2 weeks") max_dt = arrow.now().shift(weeks=-2) nb_deleted = EmailLog.filter(EmailLog.created_at < max_dt).delete() Session.commit() LOG.i("Delete %s email logs", nb_deleted)
def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]: """return whether an email has been delivered and the smtp status ("250 Message accepted", "550 Non-existent email address", etc) """ address = rcpt_to.lower().strip() # alias@SL alias = Alias.get_by(email=address) if not alias: LOG.d("alias %s not exist. Try to see if it can be created on the fly", address) alias = try_auto_create(address) if not alias: LOG.d("alias %s cannot be created on-the-fly, return 550", address) return [(False, "550 SL E3")] contact = get_or_create_contact(msg["From"], envelope.mail_from, alias) email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id) if not alias.enabled: LOG.d("%s is disabled, do not forward", alias) email_log.blocked = True db.session.commit() # do not return 5** to allow user to receive emails later when alias is enabled return [(True, "250 Message accepted for delivery")] user = alias.user ret = [] for mailbox in alias.mailboxes: ret.append( forward_email_to_mailbox(alias, msg, email_log, contact, envelope, smtp, mailbox, user)) return ret