def _update_returning_user(
user_data: user_pb2.User, force_update: bool = False, has_set_email: bool = False) \
-> timestamp_pb2.Timestamp:
if user_data.HasField('requested_by_user_at_date'):
start_of_day = now.get().replace(hour=0,
minute=0,
second=0,
microsecond=0)
if user_data.requested_by_user_at_date.ToDatetime() >= start_of_day:
if force_update:
_save_low_level(user_data)
return user_data.requested_by_user_at_date
else:
last_connection = timestamp_pb2.Timestamp()
last_connection.CopyFrom(user_data.requested_by_user_at_date)
else:
last_connection = user_data.registered_at
if user_data.profile.email:
user_data.hashed_email = auth.hash_user_email(user_data.profile.email)
if has_set_email:
base_url = parse.urljoin(flask.request.base_url, '/')[:-1]
advisor.maybe_send_late_activation_emails(
user_data, flask.current_app.config['DATABASE'], base_url)
user_data.requested_by_user_at_date.FromDatetime(now.get())
# No need to pollute our DB with super precise timestamps.
user_data.requested_by_user_at_date.nanos = 0
_save_low_level(user_data)
return last_connection
def save_user(user_data: user_pb2.User) -> user_pb2.User:
"""Save a user in the database."""
unused_, users_database, unused_ = mongo.get_connections_from_env()
users_database = users_database.with_prefix('jobflix_')
collection = users_database.user
if user_data.profile.email:
if db_user := collection.find_one(
{'hashedEmail': (hashed_email := auth.hash_user_email(user_data.profile.email))},
{'_id': 1, 'projects': 1}):
user_data.user_id = str(db_user['_id'])
new_projects = list(user_data.projects[:])
user_data.ClearField('projects')
user_data.projects.extend(
proto.create_from_mongo(p, project_pb2.Project, always_create=True)
for p in db_user.get('projects', []))
old_project_ids = {p.project_id for p in user_data.projects}
user_data.projects.extend(
p for p in new_projects if _make_project_id(p) not in old_project_ids)
elif user_data.user_id:
collection.update_one({'_id': objectid.ObjectId(user_data.user_id)}, {'$set': {
'profile.email': user_data.profile.email,
'hashedEmail': hashed_email,
}})
def simulate_coaching_emails(
user_proto: user_pb2.User, *,
database: mongo.NoPiiMongoDatabase,
attempts: int = 200, retry_after_fail: bool = False) -> Iterator[email_pb2.EmailSent]:
"""Compute the email coaching schedule."""
instant = now.get()
# Complete the user's proto with mandatory fields.
if not user_proto.HasField('registered_at'):
user_proto.registered_at.FromDatetime(instant)
if not user_proto.profile.coaching_email_frequency:
user_proto.profile.coaching_email_frequency = email_pb2.EMAIL_MAXIMUM
if not user_proto.projects:
user_proto.projects.add()
for attempt in range(attempts):
campaign_id = send_focus_email_to_user(
'ghost', user_proto, database=database, instant=instant)
if attempt and not campaign_id and not retry_after_fail:
# No more email to send.
# Note that the first call might not return a campaign ID as we do not send focus emails
# right away.
return
if campaign_id:
yield user_proto.emails_sent[-1]
instant = user_proto.send_coaching_email_after.ToDatetime()
if instant.hour > _FOCUS_EMAIL_SENDING_TIME or \
instant.hour == _FOCUS_EMAIL_SENDING_TIME and instant.minute:
instant += datetime.timedelta(days=1)
instant = instant.replace(hour=9, minute=0)
def delete_user(user_proto: user_pb2.User,
user_database: mongo.UsersDatabase) -> bool:
"""Close a user's account.
We assume the given user_proto is up-to-date, e.g. just being fetched from database.
"""
try:
user_id = objectid.ObjectId(user_proto.user_id)
except bson.errors.InvalidId:
logging.exception('Tried to delete a user with invalid ID "%s"',
user_proto.user_id)
return False
filter_user = {'_id': user_id}
# Remove authentication informations.
user_database.user_auth.delete_one(filter_user)
try:
privacy.redact_proto(user_proto)
except TypeError:
logging.exception('Cannot delete account %s', str(user_id))
return False
user_proto.deleted_at.FromDatetime(now.get())
user_proto.ClearField('user_id')
user_database.user.replace_one(filter_user,
json_format.MessageToDict(user_proto))
return True
def _copy_unmodifiable_fields(previous_user_data: user_pb2.User,
user_data: user_pb2.User) -> None:
"""Copy unmodifiable fields.
Some fields cannot be changed by the API: we only copy over the fields
from the previous state.
"""
if _is_test_user(user_data):
# Test users can do whatever they want.
return
for field in ('features_enabled', 'last_email_sent_at'):
typed_field = typing.cast(
Literal['features_enabled', 'last_email_sent_at'], field)
if previous_user_data.HasField(typed_field):
getattr(user_data, typed_field).CopyFrom(
getattr(previous_user_data, typed_field))
else:
user_data.ClearField(typed_field)
def _compute_user_data(self, user: user_pb2.User) -> Dict[str, Any]:
if not user.HasField('registered_at'):
user.registered_at.GetCurrentTime()
self._user_db.user.drop()
self._user_db.user.insert_one(json_format.MessageToDict(user))
sync_user_elasticsearch.main(self.mock_elasticsearch, [
'--registered-from', '2017-07', '--no-dry-run', '--disable-sentry'
])
kwargs = self.mock_elasticsearch.create.call_args[1]
return typing.cast(Dict[str, Any], json.loads(kwargs.pop('body')))
def _save_low_level(user_data: user_pb2.User,
is_new_user: bool = False) -> user_pb2.User:
user_collection = flask.current_app.config['USER_DATABASE'].user
user_dict = json_format.MessageToDict(user_data)
user_dict.update(SERVER_TAG)
user_dict.pop('userId', None)
if is_new_user:
result = user_collection.insert_one(user_dict)
user_data.user_id = str(result.inserted_id)
else:
user_collection.replace_one({'_id': safe_object_id(user_data.user_id)},
user_dict)
return user_data
def save_low_level(
user_data: user_pb2.User,
*,
is_new_user: bool = False,
collection: Optional[pymongo.Collection] = None) -> user_pb2.User:
"""Save the user almost 'as is' in database."""
user_collection = collection or _get_user_db().user
user_dict = json_format.MessageToDict(user_data) | SERVER_TAG
user_dict.pop('userId', None)
if is_new_user:
result = user_collection.insert_one(user_dict)
user_data.user_id = str(result.inserted_id)
else:
user_collection.replace_one({'_id': safe_object_id(user_data.user_id)},
user_dict)
return user_data
def change_email(self, user_proto: user_pb2.User, auth_request: auth_pb2.AuthRequest) \
-> user_pb2.User:
"""Change user's email address."""
new_hashed_email = hash_user_email(auth_request.email)
if user_proto.hashed_email == new_hashed_email:
# Trying to set the same email.
return user_proto
user_auth_dict = self._user_db.user_auth.find_one(
{'_id': objectid.ObjectId(user_proto.user_id)})
if user_auth_dict or auth_request.hashed_password:
if not user_auth_dict:
flask.abort(
422,
i18n.flask_translate(
"L'utilisateur n'a pas encore de mot de passe"))
stored_hashed_password = user_auth_dict.get('hashedPassword', '')
_check_password(stored_hashed_password, auth_request.hash_salt,
auth_request.hashed_password)
existing = self._user_collection.find_one(
{'hashedEmail': new_hashed_email}, {'_id': 1})
if existing:
flask.abort(
403,
i18n.flask_translate(
'L\'email "{email}" est déjà utilisé par un autre compte').
format(email=auth_request.email))
user_proto.profile.email = auth_request.email
user_proto.hashed_email = new_hashed_email
if user_auth_dict:
self._user_db.user_auth.replace_one(
{'_id': objectid.ObjectId(user_proto.user_id)},
{'hashedPassword': auth_request.new_hashed_password})
self._update_returning_user(user_proto, force_update=True)
return user_proto
def send_focus_email_to_user(
action: 'campaign.Action', user: user_pb2.User, *, dry_run_email: Optional[str] = None,
database: mongo.NoPiiMongoDatabase,
users_database: Optional[mongo.UsersDatabase] = None,
eval_database: Optional[mongo.NoPiiMongoDatabase] = None,
instant: datetime.datetime,
restricted_campaigns: Optional[Set[mailjet_templates.Id]] = None) \
-> Optional[mailjet_templates.Id]:
"""Try to send a focus email to the user and returns the campaign ID."""
if user.profile.coaching_email_frequency == email_pb2.EMAIL_NONE:
return None
if not user.HasField('send_coaching_email_after'):
send_coaching_email_after = _compute_next_coaching_email_date(user)
if send_coaching_email_after > instant:
user.send_coaching_email_after.FromDatetime(send_coaching_email_after)
if user.user_id and action != 'ghost' and users_database:
users_database.user.update_one(
{'_id': objectid.ObjectId(user.user_id)}, {'$set': {
'sendCoachingEmailAfter': proto.datetime_to_json_string(
send_coaching_email_after,
),
}})
return None
# Compute next send_coaching_email_after.
next_send_coaching_email_after = instant + _compute_duration_to_next_coaching_email(user)
focus_emails_sent: Set[mailjet_templates.Id] = set()
last_focus_email_sent = None
for email_sent in user.emails_sent:
if email_sent.campaign_id not in _FOCUS_CAMPAIGNS:
continue
last_focus_email_sent = email_sent
focus_emails_sent.add(typing.cast(mailjet_templates.Id, email_sent.campaign_id))
project = scoring.ScoringProject(
user.projects[0] if user.projects else project_pb2.Project(),
user, database, instant,
)
possible_campaigns = get_possible_campaigns(database, restricted_campaigns)
campaigns_scores = {
campaign_id: (
project.score(possible_campaign.scoring_model)
if possible_campaign.scoring_model else 2
)
for campaign_id, possible_campaign in possible_campaigns.items()
}
focus_emails_project_score_zero = {
campaign for campaign, score in campaigns_scores.items() if not score
}
potential_campaigns = _shuffle(
possible_campaigns.keys() - focus_emails_sent - focus_emails_project_score_zero,
last_focus_email_sent, campaigns_scores,
user.profile.coaching_email_frequency)
for campaign_id in potential_campaigns:
if _FOCUS_CAMPAIGNS[campaign_id].send_mail(
user, database=database, users_database=users_database, eval_database=eval_database,
action=action, dry_run_email=dry_run_email,
mongo_user_update={'$set': {
'sendCoachingEmailAfter': proto.datetime_to_json_string(
next_send_coaching_email_after,
),
}}, now=instant):
user.send_coaching_email_after.FromDatetime(next_send_coaching_email_after)
return campaign_id
# No focus email was supported: it seems that we have sent all the
# ones we had. However maybe in the future we'll add more focus
# emails so let's wait the same amount of time we have waited until
# this email (this makes to wait 1 period, then 2, 4, …).
last_coaching_email_sent_at = \
_compute_last_coaching_email_date(user, user.registered_at.ToDatetime())
send_coaching_email_after = instant + (instant - last_coaching_email_sent_at)
user.send_coaching_email_after.FromDatetime(send_coaching_email_after)
if user.user_id and action != 'ghost' and users_database:
logging.debug('No more available focus email for "%s"', user.user_id)
users_database.user.update_one({'_id': objectid.ObjectId(user.user_id)}, {'$set': {
'sendCoachingEmailAfter': proto.datetime_to_json_string(send_coaching_email_after),
}})
return None
def send_focus_email_to_user(
action: 'campaign.Action', user: user_pb2.User, *, dry_run_email: Optional[str] = None,
database: pymongo.database.Database, users_database: pymongo.database.Database,
instant: datetime.datetime) -> Optional[str]:
"""Try to send a focus email to the user and returns the campaign ID."""
if not user.HasField('send_coaching_email_after'):
send_coaching_email_after = _compute_next_coaching_email_date(user)
if send_coaching_email_after > instant:
user.send_coaching_email_after.FromDatetime(send_coaching_email_after)
if user.user_id:
users_database.user.update_one(
{'_id': objectid.ObjectId(user.user_id)}, {'$set': {
'sendCoachingEmailAfter': proto.datetime_to_json_string(
send_coaching_email_after,
),
}})
return None
# Compute next send_coaching_email_after.
next_send_coaching_email_after = instant + _compute_duration_to_next_coaching_email(user)
focus_emails_sent = set()
last_focus_email_sent = None
for email_sent in user.emails_sent:
if email_sent.campaign_id not in _FOCUS_CAMPAIGNS:
continue
last_focus_email_sent = email_sent
focus_emails_sent.add(email_sent.campaign_id)
last_one_was_big = last_focus_email_sent and \
_FOCUS_CAMPAIGNS[last_focus_email_sent.campaign_id].is_big_focus
potential_campaigns = sorted(
_POTENTIAL_CAMPAIGNS - focus_emails_sent,
key=lambda c: (
1 if _FOCUS_CAMPAIGNS[c].is_big_focus == last_one_was_big else 0,
random.random(),
)
)
for campaign_id in potential_campaigns:
if _FOCUS_CAMPAIGNS[campaign_id].send_mail(
campaign_id, user, database=database, users_database=users_database, action=action,
dry_run_email=dry_run_email,
mongo_user_update={'$set': {
'sendCoachingEmailAfter': proto.datetime_to_json_string(
next_send_coaching_email_after,
),
}}, now=instant):
user.send_coaching_email_after.FromDatetime(next_send_coaching_email_after)
return campaign_id
# No focus email was supported: it seems that we have sent all the
# ones we had. However maybe in the future we'll add more focus
# emails so let's wait the same amount of time we have waited until
# this email (this makes to wait 1 period, then 2, 4, …).
last_coaching_email_sent_at = typing.cast(
datetime.datetime,
_compute_last_coaching_email_date(user, user.registered_at.ToDatetime()))
send_coaching_email_after = instant + (instant - last_coaching_email_sent_at)
user.send_coaching_email_after.FromDatetime(send_coaching_email_after)
if user.user_id and action != 'ghost':
logging.debug('No more available focus email for "%s"', user.user_id)
users_database.user.update_one({'_id': objectid.ObjectId(user.user_id)}, {'$set': {
'sendCoachingEmailAfter': proto.datetime_to_json_string(send_coaching_email_after),
}})
return None
def save_user(user_data: user_pb2.User,
is_new_user: bool,
collection: Optional[pymongo.Collection] = None,
save_project: _SaveProject = _save_project) -> user_pb2.User:
"""Save a user, updating all the necessary computed fields while doing so."""
tick.tick('Save user start')
if is_new_user:
previous_user_data = user_data
features.assign_features(user_data.features_enabled, is_new=True)
else:
tick.tick('Load old user data')
previous_user_data = get_user_data(user_data.user_id,
collection=collection)
if user_data.revision and previous_user_data.revision > user_data.revision:
# Do not overwrite newer data that was saved already: just return it.
return previous_user_data
features.assign_features(previous_user_data.features_enabled,
is_new=False)
if not previous_user_data.registered_at.seconds:
common_proto.set_date_now(user_data.registered_at)
# Disable Advisor for new users in tests.
if ADVISOR_DISABLED_FOR_TESTING:
user_data.features_enabled.advisor = features_pb2.CONTROL
user_data.features_enabled.advisor_email = features_pb2.CONTROL
elif not _is_test_user(previous_user_data):
user_data.registered_at.CopyFrom(previous_user_data.registered_at)
user_data.features_enabled.advisor = previous_user_data.features_enabled.advisor
user_data.features_enabled.strat_two = previous_user_data.features_enabled.strat_two
# TODO(pascal): Clean up those multiple populate_feature_flags floating around.
_populate_feature_flags(user_data)
for project in user_data.projects:
previous_project = next((p for p in previous_user_data.projects
if p.project_id == project.project_id),
project_pb2.Project())
save_project(project, previous_project, user_data)
if user_data.profile.coaching_email_frequency != \
previous_user_data.profile.coaching_email_frequency:
# Invalidate the send_coaching_email_after field: it will be recomputed
# by the focus email script.
user_data.ClearField('send_coaching_email_after')
# Update hashed_email field if necessary, to make sure it's consistent with profile.email. This
# must be done for all users, since (say) a Google authenticated user may try to connect with
# password, so its email hash must be indexed.
if user_data.profile.email:
user_data.hashed_email = auth.hash_user_email(user_data.profile.email)
if not is_new_user:
_assert_no_credentials_change(previous_user_data, user_data)
_copy_unmodifiable_fields(previous_user_data, user_data)
_populate_feature_flags(user_data)
user_data.revision += 1
tick.tick('Save user')
save_low_level(user_data, is_new_user=is_new_user, collection=collection)
tick.tick('Return user proto')
return user_data
def save_user(user_data: user_pb2.User, is_new_user: bool) \
-> user_pb2.User:
"""Save a user, updating all the necessary computed fields while doing so."""
_tick('Save user start')
if is_new_user:
previous_user_data = user_data
else:
_tick('Load old user data')
previous_user_data = get_user_data(user_data.user_id)
if user_data.revision and previous_user_data.revision > user_data.revision:
# Do not overwrite newer data that was saved already: just return it.
return previous_user_data
if not previous_user_data.registered_at.seconds:
user_data.registered_at.FromDatetime(now.get())
# No need to pollute our DB with super precise timestamps.
user_data.registered_at.nanos = 0
# Disable Advisor for new users in tests.
if ADVISOR_DISABLED_FOR_TESTING:
user_data.features_enabled.advisor = user_pb2.CONTROL
user_data.features_enabled.advisor_email = user_pb2.CONTROL
elif not _is_test_user(previous_user_data):
user_data.registered_at.CopyFrom(previous_user_data.registered_at)
user_data.features_enabled.advisor = previous_user_data.features_enabled.advisor
user_data.features_enabled.strat_two = previous_user_data.features_enabled.strat_two
_populate_feature_flags(user_data)
for project in user_data.projects:
previous_project = next((p for p in previous_user_data.projects
if p.project_id == project.project_id),
project_pb2.Project())
_save_project(project, previous_project, user_data)
if user_data.profile.coaching_email_frequency != \
previous_user_data.profile.coaching_email_frequency:
# Invalidate the send_coaching_email_after field: it will be recomputed
# by the focus email script.
user_data.ClearField('send_coaching_email_after')
# Update hashed_email field if necessary, to make sure it's consistent with profile.email. This
# must be done for all users, since (say) a Google authenticated user may try to connect with
# password, so its email hash must be indexed.
if user_data.profile.email:
user_data.hashed_email = auth.hash_user_email(user_data.profile.email)
if not is_new_user:
_assert_no_credentials_change(previous_user_data, user_data)
_copy_unmodifiable_fields(previous_user_data, user_data)
_populate_feature_flags(user_data)
if user_data.profile.email and not previous_user_data.profile.email:
base_url = parse.urljoin(flask.request.base_url, '/')[:-1]
advisor.maybe_send_late_activation_emails(
user_data, flask.current_app.config['DATABASE'], base_url)
user_data.revision += 1
_tick('Save user')
_save_low_level(user_data, is_new_user=is_new_user)
_tick('Return user proto')
return user_data