def test_rate_limited_forward_phase_for_alias(flask_client):
user = User.create(email="[email protected]",
password="******",
name="Test User",
activated=True)
db.session.commit()
# no rate limiting for a new alias
alias = Alias.create_new_random(user)
db.session.commit()
assert not rate_limited_for_alias(alias)
# rate limit when there's a previous activity on alias
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
)
db.session.commit()
for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_ALIAS + 1):
EmailLog.create(user_id=user.id, contact_id=contact.id)
db.session.commit()
assert rate_limited_for_alias(alias)
def test_alias_activities(flask_client):
user = User.create(email="[email protected]",
password="******",
name="Test User",
activated=True)
Session.commit()
# create api_key
api_key = ApiKey.create(user.id, "for test")
Session.commit()
alias = Alias.create_new_random(user)
Session.commit()
# create some alias log
contact = Contact.create(
website_email="*****@*****.**",
reply_email="[email protected]",
alias_id=alias.id,
user_id=alias.user_id,
)
Session.commit()
for _ in range(int(PAGE_LIMIT / 2)):
EmailLog.create(
contact_id=contact.id,
is_reply=True,
user_id=contact.user_id,
alias_id=contact.alias_id,
)
for _ in range(int(PAGE_LIMIT / 2) + 2):
EmailLog.create(
contact_id=contact.id,
blocked=True,
user_id=contact.user_id,
alias_id=contact.alias_id,
)
r = flask_client.get(
url_for("api.get_alias_activities", alias_id=alias.id, page_id=0),
headers={"Authentication": api_key.code},
)
assert r.status_code == 200
assert len(r.json["activities"]) == PAGE_LIMIT
for ac in r.json["activities"]:
assert ac["from"]
assert ac["to"]
assert ac["timestamp"]
assert ac["action"]
assert ac["reverse_alias"]
assert ac["reverse_alias_address"]
# second page, should return 1 or 2 results only
r = flask_client.get(
url_for("api.get_alias_activities", alias_id=alias.id, page_id=1),
headers={"Authentication": api_key.code},
)
assert len(r.json["activities"]) < 3
def test_greylisting_needed_forward_phase_for_mailbox(flask_client):
user = User.create(
email="[email protected]", password="******", name="Test User", activated=True
)
db.session.commit()
alias = Alias.create_new_random(user)
db.session.commit()
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
)
db.session.commit()
for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_MAILBOX + 1):
EmailLog.create(user_id=user.id, contact_id=contact.id)
db.session.commit()
EmailLog.create(user_id=user.id, contact_id=contact.id)
# Create another alias with the same mailbox
# will be greylisted as there's a previous activity on mailbox
alias2 = Alias.create_new_random(user)
db.session.commit()
assert greylisting_needed_for_mailbox(alias2)
def test_rate_limited_reply_phase(flask_client):
# no rate limiting when reply_email does not exist
assert not rate_limited_reply_phase("*****@*****.**")
user = User.create(email="[email protected]",
password="******",
name="Test User",
activated=True)
db.session.commit()
alias = Alias.create_new_random(user)
db.session.commit()
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
)
db.session.commit()
for _ in range(MAX_ACTIVITY_DURING_MINUTE_PER_ALIAS + 1):
EmailLog.create(user_id=user.id, contact_id=contact.id)
db.session.commit()
assert rate_limited_reply_phase("*****@*****.**")
def test_should_disable(flask_client):
user = User.create(
email="[email protected]",
password="******",
name="Test User",
activated=True,
include_sender_in_reverse_alias=True,
)
alias = Alias.create_new_random(user)
db.session.commit()
assert not should_disable(alias)
# create a lot of bounce on this alias
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
for _ in range(20):
EmailLog.create(user_id=user.id,
contact_id=contact.id,
commit=True,
bounced=True)
assert should_disable(alias)
# should not affect another alias
alias2 = Alias.create_new_random(user)
db.session.commit()
assert not should_disable(alias2)
def test_should_disable_bounces_every_day(flask_client):
"""if an alias has bounces every day at least 9 days in the last 10 days, disable alias"""
user = login(flask_client)
alias = Alias.create_new_random(user)
Session.commit()
assert not should_disable(alias)
# create a lot of bounce on this alias
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
for i in range(9):
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
alias_id=contact.alias_id,
commit=True,
bounced=True,
created_at=arrow.now().shift(days=-i),
)
assert should_disable(alias)
def test_should_disable_bounces_account(flask_client):
"""if an account has more than 10 bounces every day for at least 5 days in the last 10 days, disable alias"""
user = login(flask_client)
alias = Alias.create_new_random(user)
Session.commit()
# create a lot of bounces on alias
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
for day in range(6):
for _ in range(10):
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
alias_id=contact.alias_id,
commit=True,
bounced=True,
created_at=arrow.now().shift(days=-day),
)
alias2 = Alias.create_new_random(user)
assert should_disable(alias2)
def test_should_disable_bounce_consecutive_days(flask_client):
user = login(flask_client)
alias = Alias.create_new_random(user)
Session.commit()
contact = Contact.create(
user_id=user.id,
alias_id=alias.id,
website_email="*****@*****.**",
reply_email="*****@*****.**",
commit=True,
)
# create 6 bounce on this alias in the last 24h: alias is not disabled
for _ in range(6):
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
alias_id=contact.alias_id,
commit=True,
bounced=True,
)
assert not should_disable(alias)
# create 2 bounces in the last 7 days: alias should be disabled
for _ in range(2):
EmailLog.create(
user_id=user.id,
contact_id=contact.id,
alias_id=contact.alias_id,
commit=True,
bounced=True,
created_at=arrow.now().shift(days=-3),
)
assert should_disable(alias)
def send_email(email_log_id):
email = EmailLog[email_log_id]
rv = mailext.send_email(email.to, email.subject, email.body)
email_id = rv['id']
EmailLog.set_by_id(
email_log_id, {
'email_id': email_id,
'status': EmailLog.SEND_MAILGUN,
'updated_at': datetime.datetime.now()
})
def compute_metric2() -> Metric2:
now = arrow.now()
_24h_ago = now.shift(days=-1)
nb_referred_user_paid = 0
for user in User.filter(User.referral_id.isnot(None)):
if user.is_paid():
nb_referred_user_paid += 1
return Metric2.create(
date=now,
# user stats
nb_user=User.count(),
nb_activated_user=User.filter_by(activated=True).count(),
# subscription stats
nb_premium=Subscription.filter(
Subscription.cancelled.is_(False)).count(),
nb_cancelled_premium=Subscription.filter(
Subscription.cancelled.is_(True)).count(),
# todo: filter by expires_date > now
nb_apple_premium=AppleSubscription.count(),
nb_manual_premium=ManualSubscription.filter(
ManualSubscription.end_at > now,
ManualSubscription.is_giveaway.is_(False),
).count(),
nb_coinbase_premium=CoinbaseSubscription.filter(
CoinbaseSubscription.end_at > now).count(),
# referral stats
nb_referred_user=User.filter(User.referral_id.isnot(None)).count(),
nb_referred_user_paid=nb_referred_user_paid,
nb_alias=Alias.count(),
# email log stats
nb_forward_last_24h=EmailLog.filter(
EmailLog.created_at > _24h_ago).filter_by(bounced=False,
is_spam=False,
is_reply=False,
blocked=False).count(),
nb_bounced_last_24h=EmailLog.filter(
EmailLog.created_at > _24h_ago).filter_by(bounced=True).count(),
nb_total_bounced_last_24h=Bounce.filter(
Bounce.created_at > _24h_ago).count(),
nb_reply_last_24h=EmailLog.filter(
EmailLog.created_at > _24h_ago).filter_by(is_reply=True).count(),
nb_block_last_24h=EmailLog.filter(
EmailLog.created_at > _24h_ago).filter_by(blocked=True).count(),
# other stats
nb_verified_custom_domain=CustomDomain.filter_by(
verified=True).count(),
nb_subdomain=CustomDomain.filter_by(is_sl_subdomain=True).count(),
nb_directory=Directory.count(),
nb_deleted_directory=DeletedDirectory.count(),
nb_deleted_subdomain=DeletedSubdomain.count(),
nb_app=Client.count(),
commit=True,
)
def schedule_emails():
now = datetime.datetime.now()
limit_time = now + datetime.timedelta(minutes=10)
email_objs = EmailLog.select().where(EmailLog.status == EmailLog.SCHEDULED,
EmailLog.send_at <= limit_time)
for i in email_objs:
EmailLog.set_by_id(i.id, {
'status': EmailLog.IN_QUEUE,
'updated_at': datetime.datetime.now()
})
send_email.apply_async(args=(i.id, ), eta=i.send_at)
def test_alias_contacts(flask_client):
user = User.create(
email="[email protected]", password="******", name="Test User", activated=True
)
db.session.commit()
# create api_key
api_key = ApiKey.create(user.id, "for test")
db.session.commit()
alias = Alias.create_new_random(user)
db.session.commit()
# create some alias log
for i in range(PAGE_LIMIT + 1):
contact = Contact.create(
website_email=f"marketing-{i}@example.com",
reply_email=f"reply-{i}@a.b",
alias_id=alias.id,
user_id=alias.user_id,
)
db.session.commit()
EmailLog.create(
contact_id=contact.id,
is_reply=True,
user_id=contact.user_id,
alias_id=contact.alias_id,
)
db.session.commit()
r = flask_client.get(
url_for("api.get_alias_contacts_route", alias_id=alias.id, page_id=0),
headers={"Authentication": api_key.code},
)
assert r.status_code == 200
assert len(r.json["contacts"]) == PAGE_LIMIT
for ac in r.json["contacts"]:
assert ac["creation_date"]
assert ac["creation_timestamp"]
assert ac["last_email_sent_date"]
assert ac["last_email_sent_timestamp"]
assert ac["contact"]
assert ac["reverse_alias"]
assert ac["reverse_alias_address"]
# second page, should return 1 result only
r = flask_client.get(
url_for("api.get_alias_contacts_route", alias_id=alias.id, page_id=1),
headers={"Authentication": api_key.code},
)
assert len(r.json["contacts"]) == 1
def test_delay_email(mocker, email_logs, faker):
from app.tasks import send_email
send_mock = mocker.patch.object(
send_email,
'apply_async',
)
schedule_emails()
now = datetime.datetime.now()
query = EmailLog.select().where(
EmailLog.send_at <= now + datetime.timedelta(minutes=10),
EmailLog.status != EmailLog.IN_QUEUE)
assert query.count() == 0
query = EmailLog.select().where(EmailLog.status == EmailLog.IN_QUEUE, )
assert query.count() == send_mock.call_count
def post(self):
data = request.get_json()
event_data = data['event-data']
if event_data['event'] != 'opened' or not mailext.verify(
**data['signature']):
return Response(status=406)
email_id = event_data['id']
email = EmailLog.select().where(EmailLog.email_id == email_id).first()
if email is None or email.status != EmailLog.SEND_SUCCESSFULLY:
return Response(status=406)
EmailLog.set_by_id(email.id, {
'status': EmailLog.OPENED,
'updated_at': datetime.datetime.now()
})
return Response(status=200)
def send_confirm_email(self, user):
tracking_code = email_service.confirm_email(user)
log = EmailLog(user_id=user.id,
type="confirm_email",
tracking_code=tracking_code)
db.session.add(log)
db.session.commit()
def test_email_track_open_successfully(client, mocker, faker,
email_logs_called_mailgun):
from app.api.views import mailext
verify_mock = mocker.patch.object(mailext, 'verify', return_val=True)
email = EmailLog.select().first()
EmailLog.set_by_id(email.id, {
'status': EmailLog.SEND_SUCCESSFULLY,
})
resp = client.post('/email/open-track',
json={
'event-data': {
'event': 'opened',
'id': email.email_id,
},
'signature': {}
})
assert resp.status_code == 200
assert EmailLog[email.id].status == EmailLog.OPENED
assert verify_mock.called
async def handle_forward(envelope, smtp: SMTP, msg: Message,
rcpt_to: str) -> List[Tuple[bool, str]]:
"""return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
address = rcpt_to.lower().strip() # alias@SL
alias = Alias.get_by(email=address)
if not alias:
LOG.d("alias %s not exist. Try to see if it can be created on the fly",
address)
alias = try_auto_create(address)
if not alias:
LOG.d("alias %s cannot be created on-the-fly, return 550", address)
return [(False, "550 SL E3 Email not exist")]
mail_from = envelope.mail_from.lower().strip()
for mb in alias.mailboxes:
# email send from a mailbox to alias
if mb.email.lower().strip() == mail_from:
LOG.exception("cycle email sent from %s to %s", mb, alias)
handle_email_sent_to_ourself(alias, mb, msg, alias.user)
return [(True, "250 Message accepted for delivery")]
contact = get_or_create_contact(msg["From"], envelope.mail_from, alias)
email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id)
db.session.commit()
if not alias.enabled:
LOG.d("%s is disabled, do not forward", alias)
email_log.blocked = True
db.session.commit()
# do not return 5** to allow user to receive emails later when alias is enabled
return [(True, "250 Message accepted for delivery")]
user = alias.user
ret = []
mailboxes = alias.mailboxes
# no need to create a copy of message
if len(mailboxes) == 1:
mailbox = mailboxes[0]
ret.append(await
forward_email_to_mailbox(alias, msg, email_log, contact,
envelope, smtp, mailbox, user))
# create a copy of message for each forward
else:
for mailbox in mailboxes:
ret.append(await
forward_email_to_mailbox(alias, copy(msg), email_log,
contact, envelope, smtp,
mailbox, user))
return ret
def test_send_email(mocker, email_logs, faker):
from app.tasks import mailext
return_val = {'id': faker.isbn13()}
send_mock = mocker.patch.object(mailext,
'send_email',
return_value=return_val)
email = EmailLog.select().first()
send_email(email.id)
assert send_mock.called == 1
assert EmailLog[email.id].status == EmailLog.SEND_MAILGUN
assert EmailLog[email.id].email_id == return_val['id']
def test_alias_contacts(flask_client):
user = login(flask_client)
alias = Alias.create_new_random(user)
Session.commit()
# create some alias log
for i in range(PAGE_LIMIT + 1):
contact = Contact.create(
website_email=f"marketing-{i}@example.com",
reply_email=f"reply-{i}@a.b",
alias_id=alias.id,
user_id=alias.user_id,
)
Session.commit()
EmailLog.create(
contact_id=contact.id,
is_reply=True,
user_id=contact.user_id,
alias_id=contact.alias_id,
)
Session.commit()
r = flask_client.get(f"/api/aliases/{alias.id}/contacts?page_id=0")
assert r.status_code == 200
assert len(r.json["contacts"]) == PAGE_LIMIT
for ac in r.json["contacts"]:
assert ac["creation_date"]
assert ac["creation_timestamp"]
assert ac["last_email_sent_date"]
assert ac["last_email_sent_timestamp"]
assert ac["contact"]
assert ac["reverse_alias"]
assert ac["reverse_alias_address"]
assert "block_forward" in ac
# second page, should return 1 result only
r = flask_client.get(f"/api/aliases/{alias.id}/contacts?page_id=1")
assert len(r.json["contacts"]) == 1
def post(self):
data = request.get_json()
event_data = data['event-data']
if event_data['event'] not in (
'delivered',
'failed',
) or not mailext.verify(**data['signature']):
return Response(status=406)
email_id = event_data['id']
email = EmailLog.select().where(EmailLog.email_id == email_id).first()
if email is None or email.status != EmailLog.SEND_MAILGUN:
return Response(status=406)
if event_data['event'] == 'failed':
EmailLog.set_by_id(email.id, {'status': EmailLog.SEND_FAILED})
else:
EmailLog.set_by_id(
email.id, {
'status': EmailLog.SEND_SUCCESSFULLY,
'updated_at': datetime.datetime.now()
})
return Response(status=200)
def forgot_password():
request_data = request.get_json()
email = request_data['email']
user = User.query.filter(
func.lower(User.email) == email.lower()).first_or_404()
tracking_code = email_service.reset_email(user)
log = EmailLog(user_id=user.id,
type="reset_email",
tracking_code=tracking_code)
db.session.add(log)
db.session.commit()
return ''
def approval_request():
request_data = request.get_json()
user_id = request_data['user_id']
resource_id = request_data['resource_id']
resource = ThrivResource.query.filter_by(id=resource_id).first_or_404()
# Get admins for the user's institution
user = User.query.filter_by(id=user_id).first_or_404()
admins = User.query.filter_by(role="Admin", institution_id=user.institution_id).all()
# Send approval request email to each admin for the institution
for admin in admins:
tracking_code = email_service.approval_request_email(user, admin, resource)
log = EmailLog(user_id=admin.id, type="approval_request", tracking_code=tracking_code)
db.session.add(log)
db.session.commit()
# Send confirmation email to user
confirm_tracking_code = email_service.approval_request_confirm_email(user, resource)
confirm_log = EmailLog(user_id=user.id, type="approval_request_confirm", tracking_code=confirm_tracking_code)
db.session.add(confirm_log)
db.session.commit()
return ''
def test_schedule_email_by_countdown(client, faker):
data = {
'to': faker.email(),
'countdown': {
'hours': faker.pyint(),
'minutes': faker.pyint(),
'seconds': faker.pyint(),
},
'body': faker.text(),
'subject': faker.sentence()
}
resp = client.post('/email/schedule', json=data)
assert resp.status_code == 200
email = EmailLog.select().first()
assert email.created_at + datetime.timedelta(
**data['countdown']) == email.send_at
def test_email_track_send_failed(client, faker, mocker,
email_logs_called_mailgun):
from app.api.views import mailext
verify_mock = mocker.patch.object(mailext, 'verify', return_val=True)
email = EmailLog.select().first()
resp = client.post('/email/send-track',
json={
'event-data': {
'event': 'failed',
'id': email.email_id,
},
'signature': {}
})
assert resp.status_code == 200
assert EmailLog[email.id].status == EmailLog.SEND_FAILED
assert verify_mock.called
def consult_request():
request_data = request.get_json()
user_id = request_data['user_id']
user = User.query.filter_by(id=user_id).first_or_404()
admins = User.query.filter_by(role="Admin",
institution_id=user.institution_id).all()
# Send consult request email to each admin for the institution
for admin in admins:
tracking_code = email_service.consult_email(user, admin, request_data)
log = EmailLog(user_id=admin.id,
type="consult_request",
tracking_code=tracking_code)
db.session.add(log)
db.session.commit()
return ''
def test_schedule_email_by_eta(client, mocker, faker):
from app.tasks import send_email
mocker.patch.object(
send_email,
'apply_async',
)
data = {
'to': faker.email(),
'eta': str(faker.future_datetime()),
'body': faker.text(),
'subject': faker.sentence()
}
resp = client.post('/email/schedule', json=data)
assert resp.status_code == 200
email = EmailLog.select().first()
assert email.to == data['to']
assert str(email.send_at) == data['eta']
assert email.body == data['body']
def fill_up_email_log_alias():
"""Fill up email_log.alias_id column"""
# split all emails logs into 1000-size trunks
nb_email_log = EmailLog.count()
LOG.d("total trunks %s", nb_email_log // 1000 + 2)
for trunk in reversed(range(1, nb_email_log // 1000 + 2)):
nb_update = 0
for email_log, contact in (Session.query(
EmailLog,
Contact).filter(EmailLog.contact_id == Contact.id).filter(
EmailLog.id <= trunk * 1000).filter(
EmailLog.id > (trunk - 1) * 1000).filter(
EmailLog.alias_id.is_(None))):
email_log.alias_id = contact.alias_id
nb_update += 1
LOG.d("finish trunk %s, update %s email logs", trunk, nb_update)
Session.commit()
def post(self):
data = request.get_json()
now = datetime.datetime.now()
if data.get('eta') is not None:
send_at = datetime.datetime.strptime(data['eta'],
'%Y-%m-%d %H:%M:%S')
if send_at < now:
raise ScheduleTimeException(
'schedule time:{} error'.format(send_at))
else:
countdown = data['countdown']
send_at = datetime.datetime.now() + datetime.timedelta(**countdown)
email = EmailLog.create(subject=data['subject'],
body=data['body'],
to=data['to'],
send_at=send_at)
if send_at - now <= datetime.timedelta(minutes=15):
send_email.apply_async(args=(email.id, ), eta=send_at)
return jsonify()
def delete_logs():
"""delete everything that are considered logs"""
delete_refused_emails()
delete_old_monitoring()
for t in TransactionalEmail.filter(
TransactionalEmail.created_at < arrow.now().shift(days=-7)):
TransactionalEmail.delete(t.id)
for b in Bounce.filter(Bounce.created_at < arrow.now().shift(days=-7)):
Bounce.delete(b.id)
Session.commit()
LOG.d("Delete EmailLog older than 2 weeks")
max_dt = arrow.now().shift(weeks=-2)
nb_deleted = EmailLog.filter(EmailLog.created_at < max_dt).delete()
Session.commit()
LOG.i("Delete %s email logs", nb_deleted)
def handle_forward(envelope, smtp: SMTP, msg: Message,
rcpt_to: str) -> List[Tuple[bool, str]]:
"""return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
address = rcpt_to.lower().strip() # alias@SL
alias = Alias.get_by(email=address)
if not alias:
LOG.d("alias %s not exist. Try to see if it can be created on the fly",
address)
alias = try_auto_create(address)
if not alias:
LOG.d("alias %s cannot be created on-the-fly, return 550", address)
return [(False, "550 SL E3")]
contact = get_or_create_contact(msg["From"], envelope.mail_from, alias)
email_log = EmailLog.create(contact_id=contact.id, user_id=contact.user_id)
if not alias.enabled:
LOG.d("%s is disabled, do not forward", alias)
email_log.blocked = True
db.session.commit()
# do not return 5** to allow user to receive emails later when alias is enabled
return [(True, "250 Message accepted for delivery")]
user = alias.user
ret = []
for mailbox in alias.mailboxes:
ret.append(
forward_email_to_mailbox(alias, msg, email_log, contact, envelope,
smtp, mailbox, user))
return ret
