def instantiate(
action: action_pb2.Action,
user_proto: user_pb2.User,
project: project_pb2.Project,
template: action_pb2.ActionTemplate,
base: mongo.NoPiiMongoDatabase) -> action_pb2.Action:
"""Instantiate a newly created action from a template.
Args:
action: the action to be populated from the template.
user_proto: the whole user data.
project: the whole project data.
template: the action template to instantiate.
base: a MongoDB client to get stats and info.
Returns:
the populated action for chaining.
"""
action.action_id = f'{project.project_id}-{template.action_template_id}-' \
f'{round(time.time()):x}-{random.randrange(0x10000):x}'
action.action_template_id = template.action_template_id
action.title = template.title
action.short_description = template.short_description
scoring_project = scoring.ScoringProject(project, user_proto, base)
action.link = scoring_project.populate_template(template.link)
action.how_to = template.how_to
action.status = action_pb2.ACTION_UNREAD
action.created_at.FromDatetime(now.get())
action.image_url = template.image_url
return action
def _send_focus_emails(
action: 'campaign.NoGhostAction', dry_run_email: str,
restricted_campaigns: Optional[Iterable[mailjet_templates.Id]] = None) -> None:
database, users_database, eval_database = mongo.get_connections_from_env()
instant = now.get()
email_errors = 0
counts = {
campaign_id: 0
for campaign_id in sorted(get_possible_campaigns(database, restricted_campaigns))
}
potential_users = users_database.user.find({
'profile.email': {
'$regex': re.compile(r'[^ ]+@[^ ]+\.[^ ]+'),
'$not': re.compile(r'@example.com$'),
},
'projects': {'$elemMatch': {
'isIncomplete': {'$ne': True},
}},
'profile.coachingEmailFrequency': {'$in': [
email_pb2.EmailFrequency.Name(setting) for setting in _EMAIL_PERIOD_DAYS]},
# Note that "not >" is not equivalent to "<=" in the case the field
# is not defined: in that case we do want to select the user.
'sendCoachingEmailAfter': {'$not': {'$gt': proto.datetime_to_json_string(instant)}},
})
restricted_campaigns_set: Optional[Set[mailjet_templates.Id]]
if restricted_campaigns:
restricted_campaigns_set = set(restricted_campaigns)
else:
restricted_campaigns_set = None
for user_dict in potential_users:
user_id = user_dict.pop('_id')
user = proto.create_from_mongo(user_dict, user_pb2.User)
user.user_id = str(user_id)
try:
campaign_id = send_focus_email_to_user(
action, user, dry_run_email=dry_run_email, database=database,
users_database=users_database, eval_database=eval_database, instant=instant,
restricted_campaigns=restricted_campaigns_set)
except requests.exceptions.HTTPError as error:
if action == 'dry-run':
raise
logging.warning('Error while sending an email: %s', error)
email_errors += 1
continue
if campaign_id:
counts[campaign_id] += 1
if action == 'dry-run':
break
continue
report_message = 'Focus emails sent today:\n' + '\n'.join([
f' • *{campaign_id}*: {count} email{"s" if count > 1 else ""}'
for campaign_id, count in counts.items()
])
if action == 'send':
report.notify_slack(report_message)
logging.info(report_message)
def main(string_args: Optional[list[str]] = None) -> None:
"""Clean all support tickets marked for deletion."""
user_db = mongo.get_connections_from_env().user_db
parser = argparse.ArgumentParser(
description='Clean support tickets from the database.')
report.add_report_arguments(parser)
args = parser.parse_args(string_args)
if not report.setup_sentry_logging(args):
return
instant = proto.datetime_to_json_string(now.get())
result = user_db.user.update_many(
{}, {'$pull': {
'supportTickets': {
'deleteAfter': {
'$lt': instant
}
}
}})
logging.info('Removed deprecated support tickets for %d users.',
result.modified_count)
clean_result = user_db.user.update_many({'supportTickets': {
'$size': 0
}}, {'$unset': {
'supportTickets': ''
}})
if clean_result.matched_count:
logging.info('Removed empty support ticket list for %d users.',
clean_result.modified_count)
def _update_email_sent_status(
email_sent_dict: dict[str, Any],
yesterday: str,
campaign_ids: Optional[list[str]] = None) -> dict[str, Any]:
email_sent = proto.create_from_mongo(email_sent_dict, email_pb2.EmailSent)
if campaign_ids and email_sent.campaign_id not in campaign_ids:
# Email is not from a campaign we wish to update, skipping.
return email_sent_dict
if email_sent.status != email_pb2.EMAIL_SENT_UNKNOWN and email_sent.last_status_checked_at:
sent_at = email_sent.sent_at.ToJsonString()
if sent_at < yesterday:
last_status_checked_at = email_sent.last_status_checked_at.ToJsonString(
)
if email_sent.last_status_checked_after_days > 14 or last_status_checked_at > yesterday:
return email_sent_dict
message = _find_message(email_sent)
if message:
email_sent.mailjet_message_id = message.get(
'ID', email_sent.mailjet_message_id)
status = message.get('Status')
if status:
email_sent.status = email_pb2.EmailSentStatus.Value(
f'EMAIL_SENT_{status.upper()}')
else:
logging.warning('No status for message "%s"',
email_sent.mailjet_message_id)
else:
logging.warning('Could not find a message in MailJet.')
common_proto.set_date_now(email_sent.last_status_checked_at)
email_sent.last_status_checked_after_days = (
now.get() - email_sent.sent_at.ToDatetime()).days
return json_format.MessageToDict(email_sent)
def simulate_coaching_emails(
user_proto: user_pb2.User, *,
database: mongo.NoPiiMongoDatabase,
attempts: int = 200, retry_after_fail: bool = False) -> Iterator[email_pb2.EmailSent]:
"""Compute the email coaching schedule."""
instant = now.get()
# Complete the user's proto with mandatory fields.
if not user_proto.HasField('registered_at'):
user_proto.registered_at.FromDatetime(instant)
if not user_proto.profile.coaching_email_frequency:
user_proto.profile.coaching_email_frequency = email_pb2.EMAIL_MAXIMUM
if not user_proto.projects:
user_proto.projects.add()
for attempt in range(attempts):
campaign_id = send_focus_email_to_user(
'ghost', user_proto, database=database, instant=instant)
if attempt and not campaign_id and not retry_after_fail:
# No more email to send.
# Note that the first call might not return a campaign ID as we do not send focus emails
# right away.
return
if campaign_id:
yield user_proto.emails_sent[-1]
instant = user_proto.send_coaching_email_after.ToDatetime()
if instant.hour > _FOCUS_EMAIL_SENDING_TIME or \
instant.hour == _FOCUS_EMAIL_SENDING_TIME and instant.minute:
instant += datetime.timedelta(days=1)
instant = instant.replace(hour=9, minute=0)
def delete_user(user_proto: user_pb2.User,
user_database: mongo.UsersDatabase) -> bool:
"""Close a user's account.
We assume the given user_proto is up-to-date, e.g. just being fetched from database.
"""
try:
user_id = objectid.ObjectId(user_proto.user_id)
except bson.errors.InvalidId:
logging.exception('Tried to delete a user with invalid ID "%s"',
user_proto.user_id)
return False
filter_user = {'_id': user_id}
# Remove authentication informations.
user_database.user_auth.delete_one(filter_user)
try:
privacy.redact_proto(user_proto)
except TypeError:
logging.exception('Cannot delete account %s', str(user_id))
return False
user_proto.deleted_at.FromDatetime(now.get())
user_proto.ClearField('user_id')
user_database.user.replace_one(filter_user,
json_format.MessageToDict(user_proto))
return True
def main(string_args: Optional[list[str]] = None) -> None:
"""Parse command line arguments and trigger the update_users_client_metrics function."""
parser = argparse.ArgumentParser(
description='Synchronize MongoDB client metrics fields from Amplitude')
parser.add_argument(
'--registered-from',
help='Consider only users who registered after this date.')
yesterday = str((now.get() - datetime.timedelta(days=1)).date())
parser.add_argument(
'--registered-to',
default=yesterday,
help='Consider only users who registered before this date.')
report.add_report_arguments(parser)
args = parser.parse_args(string_args)
if not report.setup_sentry_logging(args):
return
user_db = mongo.get_connections_from_env().user_db
update_users_client_metrics(user_db.user,
from_date=args.registered_from,
to_date=args.registered_to,
dry_run=args.dry_run)
def _compute_data_bob_has_helped(self, created_at: datetime.datetime,
bob_has_helped: str) -> dict[str, Any]:
user = user_pb2.User()
user.registered_at.FromDatetime(now.get())
status = user.employment_status.add()
status.bob_has_helped = bob_has_helped
status.created_at.FromDatetime(created_at)
return self._compute_user_data(user)
def get_scoring_project(user_id: str,
project_id: str) -> scoring.ScoringProject:
"""Get the scoring project or abort."""
user_proto = get_user_data(user_id)
project = get_project_data(user_proto, project_id)
return scoring.ScoringProject(project,
user_proto,
mongo.get_connections_from_env().stats_db,
now=now.get())
def _ensure_cache(self) -> dict[str, _Type]:
instant = now.get()
if self._cached_valid_until and self._cached_valid_until >= instant and \
self._cache_version >= self._global_cache_version and self._cache is not None:
return self._cache
values = self._get_values()
self._cache = values
self._cache_version = self._global_cache_version
self._cached_valid_until = instant + self._cache_duration
return values
def _log_request(email: str, requester_email: str, database: mongo.NoPiiMongoDatabase) -> None:
try:
# Log that we've tried to access to a specific user.
database.email_requests.insert_one({
'email': email,
'registeredAt': proto.datetime_to_json_string(now.get()),
'requesterEmail': requester_email,
'action': 'eval',
})
except errors.OperationFailure:
flask.abort(401, "Vous n'avez pas accès en écriture à la base de données.")
def test_search_for_quite_long_time(self) -> None:
"""User searching for 7 months should have a low score."""
if self.persona.user_profile.year_of_birth <= datetime.date.today(
).year - 45:
self.persona.user_profile.year_of_birth = datetime.date.today(
).year - 40
self.persona.project.job_search_has_not_started = False
self.persona.project.job_search_started_at.FromDatetime(
now.get() - datetime.timedelta(days=214))
score = self._score_persona(self.persona)
self.assertEqual(score, 1, msg=f'Failed for "{self.persona.name}"')
def test_search_reasonable_time(self) -> None:
"""User searching for 4 months should have a 0 score."""
self.persona.project.diagnostic.ClearField('category_id')
if self.persona.user_profile.year_of_birth <= datetime.date.today(
).year - 45:
self.persona.user_profile.year_of_birth = datetime.date.today(
).year - 40
self.persona.project.job_search_has_not_started = False
self.persona.project.job_search_started_at.FromDatetime(
now.get() - datetime.timedelta(days=124))
score = self._score_persona(self.persona)
self.assertEqual(score, 0, msg=f'Failed for "{self.persona.name}"')
def test_passionate_search_just_started(self) -> None:
"""User passionate about their job and searching for 15 days should have a low score."""
if self.persona.user_profile.year_of_birth <= datetime.date.today(
).year - 45:
self.persona.user_profile.year_of_birth = datetime.date.today(
).year - 40
self.persona.project.passionate_level = project_pb2.LIFE_GOAL_JOB
self.persona.project.job_search_has_not_started = False
self.persona.project.job_search_started_at.FromDatetime(
now.get() - datetime.timedelta(days=15))
score = self._score_persona(self.persona)
self.assertEqual(score, 1, msg=f'Failed for "{self.persona.name}"')
def test_search_for_very_long_time(self) -> None:
"""User searching for 13 months should have a high score."""
if self.persona.user_profile.year_of_birth <= datetime.date.today(
).year - 45:
self.persona.user_profile.year_of_birth = datetime.date.today(
).year - 40
self.persona.project.job_search_has_not_started = False
self.persona.project.job_search_started_at.FromDatetime(
now.get() - datetime.timedelta(days=397))
project = self.persona.scoring_project(self.database, now=self.now)
score, explanations = self.model.score_and_explain(project)
self.assertEqual(score, 3, msg=f'Failed for "{self.persona.name}"')
self.assertEqual(['vous cherchez depuis 13 mois'], explanations)
def test_search_just_started(self) -> None:
"""User searching for 15 days should have a medium score."""
if self.persona.user_profile.year_of_birth <= datetime.date.today(
).year - 45:
self.persona.user_profile.year_of_birth = datetime.date.today(
).year - 40
if self.persona.project.passionate_level >= project_pb2.PASSIONATING_JOB:
self.persona.project.passionate_level = project_pb2.ALIMENTARY_JOB
self.persona.project.job_search_has_not_started = False
self.persona.project.job_search_started_at.FromDatetime(
now.get() - datetime.timedelta(days=15))
score = self._score_persona(self.persona)
self.assertEqual(score, 2, msg=f'Failed for "{self.persona.name}"')
def _save_project(
project: project_pb2.Project, unused_previous_project: project_pb2.Project,
user_data: user_pb2.User) -> project_pb2.Project:
database, users_database, eval_database = mongo.get_connections_from_env()
users_database = users_database.with_prefix('jobflix_')
project.project_id = _make_project_id(project)
if not project.HasField('created_at'):
common_proto.set_date_now(project.created_at)
if user_data.profile.email:
all_campaigns.send_campaign(
'jobflix-welcome', user_data, action='send',
database=database, users_database=users_database, eval_database=eval_database,
now=now.get())
_give_coaching_feedback(user_data.user_id, user_data.profile.email, project)
return project
def _maybe_recommend_advice(user: user_pb2.User, project: project_pb2.Project,
database: mongo.NoPiiMongoDatabase) -> bool:
if user.features_enabled.advisor == features_pb2.CONTROL:
return False
scoring_project = scoring.ScoringProject(project,
user,
database,
now=now.get())
if user.features_enabled.action_plan == features_pb2.ACTIVE and not project.actions:
compute_actions_for_project(scoring_project)
if project.advices:
return False
advices = compute_advices_for_project(scoring_project)
for piece_of_advice in advices.advices:
piece_of_advice.status = project_pb2.ADVICE_RECOMMENDED
project.advices.extend(advices.advices[:])
return True
def _send_activation_email(user: user_pb2.User,
project: project_pb2.Project) -> None:
"""Send an email to the user just after we have defined their diagnosis."""
database, users_database, eval_database = mongo.get_connections_from_env()
if not user.projects or user.projects[0] != project:
user_with_project = user_pb2.User()
user_with_project.CopyFrom(user)
if not user_with_project.projects:
user_with_project.projects.add()
user_with_project.projects[0].CopyFrom(project)
user = user_with_project
all_campaigns.send_campaign('activation-email',
user,
action='send',
database=database,
users_database=users_database,
eval_database=eval_database,
now=now.get())
def age_group(year_of_birth: int) -> str:
"""Estimate age group from year of birth."""
if year_of_birth < 1920:
return 'Unknown'
precise_age = now.get() - datetime.datetime(year_of_birth, 7, 1)
age = precise_age.days / 365
if age < 18:
return 'A. -18'
if age < 25:
return 'B. 18-24'
if age < 35:
return 'C. 25-34'
if age < 45:
return 'D. 35-44'
if age < 55:
return 'E. 45-54'
if age < 65:
return 'F. 55-64'
return 'G. 65+'
def test_recommendations_in_both_categories(self) -> None:
"""Users with a total of two recommended jobs should have a high score."""
self.database.local_diagnosis.drop()
self.database.local_diagnosis.insert_one({
'_id':
'09:M1601',
'imt': {
'yearlyAvgOffersPer10Candidates': 4,
},
'lessStressfulJobGroups': [{
'localStats': {
'imt': {
'yearlyAvgOffersPer10Candidates': 12
}
},
'jobGroup': {
'romeId': 'A1413',
'name': 'Aide caviste'
},
'mobilityType': job_pb2.CLOSE,
}, {
'localStats': {
'imt': {
'yearlyAvgOffersPer10Candidates': 6
}
},
'jobGroup': {
'romeId': 'A1401',
'name': 'Aide arboricole'
},
'mobilityType': job_pb2.EVOLUTION,
}],
})
self.persona.user_profile.year_of_birth = datetime.date.today(
).year - 40
self.persona.project.job_search_has_not_started = False
self.persona.project.job_search_started_at.FromDatetime(
now.get() - datetime.timedelta(days=397))
score = self._score_persona(self.persona)
self.assertEqual(score, 3, msg=f'Failed for "{self.persona.name}"')
def __call__(self, collection: str,
document_id: str) -> _MongoFindOneOutType:
cached_documents = self._cache
instant = now.get()
args = (collection, document_id)
if args in cached_documents:
result, invalidation_date = cached_documents[args]
if invalidation_date > instant:
cached_documents.move_to_end(args)
return result
result = _find_one(*args)
invalidation_date = instant + self._ttl
cached_documents[args] = result, invalidation_date
if len(cached_documents) > self._max_size:
for key in cached_documents:
if cached_documents[key][1] > instant:
del cached_documents[key]
break
else:
cached_documents.popitem(last=False)
return result
def diagnose(
user: user_pb2.User, project: project_pb2.Project, database: mongo.NoPiiMongoDatabase) \
-> diagnostic_pb2.Diagnostic:
"""Diagnose a project.
Args:
user: the user's data, mainly used for their profile and features_enabled.
project: the project data. It will be modified, as its diagnostic field
will be populated.
database: access to the MongoDB with market data.
Returns:
the modified diagnostic protobuf.
"""
diagnostic = project.diagnostic
scoring_project = scoring.ScoringProject(project,
user,
database,
now=now.get())
return diagnose_scoring_project(scoring_project, diagnostic)
def list_all_tips(
user: user_pb2.User, project: project_pb2.Project,
piece_of_advice: project_pb2.Advice,
database: mongo.NoPiiMongoDatabase) -> list[action_pb2.ActionTemplate]:
"""List all available tips for a piece of advice.
Args:
user: the full user info.
project: the project to give tips for.
piece_of_advice: the piece of advice to give tips for.
database: access to the database to get modules and tips.
Returns:
An iterable of tips for this module.
"""
try:
module = next(m for m in _advice_modules(database)
if m.advice_id == piece_of_advice.advice_id)
except StopIteration:
logging.warning('Advice module %s does not exist anymore',
piece_of_advice.advice_id)
return []
# Get tip templates.
all_tip_templates = _tip_templates(database)
tip_templates = filter(None, (all_tip_templates.get(t)
for t in module.tip_template_ids))
# Filter tips.
scoring_project = scoring.ScoringProject(project,
user,
database,
now=now.get())
filtered_tips = scoring.filter_using_score(tip_templates,
lambda t: t.filters,
scoring_project)
return [_translate_tip(tip, scoring_project) for tip in filtered_tips]
def __init__(self,
days_since_any_email: int = 7,
days_since_same_campaign_unread: int = 0,
days_since_same_campaign: int = 0) -> None:
"""Constructor for an EmailPolicy object.
Args:
days_since_any_email: number of days to wait before sending any new
mail to the users.
days_since_same_campaign_unread: number of days to wait before sending
again the same campaign email to a user to whom it has already been
sent and who has not read/open it. ATTENTION: emails status have
to be updated in mongodb.
days_since_same_campaign: number of days to wait before sending
again the same campaign email to a user whom it has already been
sent whether they have opened it or not.
"""
instant = now.get()
self.last_email_datetime = instant - datetime.timedelta(
days=days_since_any_email)
self.retry_campaign_date_unread: Optional[datetime.datetime]
if days_since_same_campaign_unread > 0:
self.retry_campaign_date_unread = \
instant - datetime.timedelta(days=days_since_same_campaign_unread)
else:
self.retry_campaign_date_unread = None
self.retry_campaign_date: Optional[datetime.datetime]
if days_since_same_campaign > 0:
self.retry_campaign_date = instant - datetime.timedelta(
days=days_since_same_campaign)
else:
self.retry_campaign_date = None
def __init__(self, api_key: str) -> None:
self._i18n_base = airtable.Airtable(_I18N_BASE_ID, api_key)
self._existing_translations: dict[str,
airtable.Record[Mapping[str,
Any]]] = {}
self._duplicate_strings: dict[
str, list[str]] = collections.defaultdict(list)
for record in self._i18n_base.iterate('translations'):
key = typing.cast(dict[str, Optional[str]],
record['fields']).get('string')
if not key:
continue
if key in self._existing_translations:
self._duplicate_strings[self._existing_translations[key]
['id']].append(record['id'])
continue
self._existing_translations[key] = record
self._api_key = api_key
self.bases: dict[str, airtable.Airtable] = {}
self._collected: dict[str, dict[str, str]] = \
collections.defaultdict(lambda: collections.defaultdict(str))
self._used_translations: Set[str] = set()
self._now = now.get().isoformat() + 'Z'
self._today = self._now[:len('2020-12-09')]
def blast_campaign(campaign_id: mailjet_templates.Id,
action: 'campaign.Action', registered_from: str,
registered_to: str, dry_run_email: str, user_hash: str,
user_id_start: str, collection_prefix: str,
email_policy: EmailPolicy,
log_reason_on_error: bool) -> int:
"""Send a campaign of personalized emails."""
if action == 'send' and auth_token.SECRET_SALT == auth_token.FAKE_SECRET_SALT:
raise ValueError('Set the prod SECRET_SALT env var before continuing.')
database, user_database, eval_database = mongo.get_connections_from_env()
user_database = user_database.with_prefix(collection_prefix)
this_campaign = campaign.get_campaign(campaign_id)
mongo_filters = dict(this_campaign.mongo_filters or {})
mongo_filters['profile.email'] = {
'$not': re.compile(r'@example.com$'),
'$regex': re.compile(r'[^ ]+@[^ ]+\.[^ ]+'),
}
if 'registeredAt' not in mongo_filters:
mongo_filters['registeredAt'] = {}
mongo_filters['registeredAt'].setdefault('$gt', registered_from)
mongo_filters['registeredAt'].setdefault('$lt', registered_to)
selected_users = user_database.user.find(mongo_filters)
email_count = 0
email_errors = 0
users_processed_count = 0
users_wrong_id_count = 0
users_wrong_hash_count = 0
users_stopped_seeking = 0
email_policy_rejections = 0
no_template_vars_count = 0
for user_dict in selected_users:
users_processed_count += 1
user = proto.create_from_mongo(user_dict, user_pb2.User, 'user_id')
if user_id_start and not user.user_id.startswith(user_id_start):
users_wrong_id_count += 1
continue
hash_value = _hash_user_id(user.user_id)
if user_hash and not hash_value.startswith(user_hash):
users_wrong_hash_count += 1
continue
if any(status.seeking == user_pb2.STOP_SEEKING
for status in user.employment_status):
users_stopped_seeking += 1
continue
if not email_policy.can_send(campaign_id, user.emails_sent):
email_policy_rejections += 1
continue
try:
if not this_campaign.send_mail(
user,
database=database,
users_database=user_database,
eval_database=eval_database,
action=action,
dry_run_email=dry_run_email,
now=now.get(),
should_log_errors=log_reason_on_error):
no_template_vars_count += 1
continue
except requests.exceptions.HTTPError as error:
if action == 'dry-run':
raise
logging.warning('Error while sending an email: %s', error)
email_errors += 1
continue
if action == 'dry-run':
break
email_count += 1
if email_count % 100 == 0:
print(f'{email_count} emails sent ...')
logging.info('%d users processed.', users_processed_count)
if users_wrong_id_count:
logging.info('%d users ignored because of ID selection.',
users_wrong_id_count)
if users_wrong_hash_count:
logging.info('%d users ignored because of hash selection.',
users_wrong_hash_count)
logging.info('%d users have stopped seeking.', users_stopped_seeking)
logging.info('%d users ignored because of emailing policy.',
email_policy_rejections)
logging.info('%d users ignored because of no template vars.',
no_template_vars_count)
if action == 'send':
campaign_url = this_campaign.get_sent_campaign_url()
if campaign_url:
campaign_url_message = \
f' You can check opening and click stats on <{campaign_url}|Mailjet>'
else:
campaign_url_message = ''
logging.info(
"Report for %s blast: I've sent %d emails (and got "
'%d errors).%s', campaign_id, email_count, email_errors,
campaign_url_message)
return email_count
def main(string_args: Optional[list[str]] = None) -> None:
"""Check the status of sent emails on MailJet and update our Database.
"""
parser = argparse.ArgumentParser(
description='Update email status on sent emails.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
report.add_report_arguments(parser)
parser.add_argument(
'--campaigns',
choices=mail_blast.campaign.list_all_campaigns(),
nargs='*',
help='Campaign IDs to check. If not specified, run for all campaigns.')
parser.add_argument('--mongo-collection',
default='user',
help='Name of the mongo collection to update.')
args = parser.parse_args(string_args)
if not report.setup_sentry_logging(args):
return
email_mongo_filter = {
'mailjetMessageId': {
'$exists': True
},
}
if args.campaigns:
email_mongo_filter['campaignId'] = {'$in': args.campaigns}
yesterday = proto.datetime_to_json_string(now.get() -
datetime.timedelta(days=1))
mongo_filter = {
'$or': [
# Emails that we've never checked.
{
'emailsSent': {
'$elemMatch':
dict({
'lastStatusCheckedAt': {
'$exists': False
},
}, **email_mongo_filter),
},
},
# Emails checked less than two weeks after they have been sent and
# that we haven't checked today.
{
'emailsSent': {
'$elemMatch':
dict(
{
'lastStatusCheckedAt': {
'$lt': yesterday
},
'lastStatusCheckedAfterDays': {
'$not': {
'$gte': 14
}
},
}, **email_mongo_filter),
},
},
# Emails sent less than 24 hours ago.
{
'emailsSent': {
'$elemMatch':
dict({
'sentAt': {
'$gt': yesterday
},
}, **email_mongo_filter),
},
},
],
}
user_db = mongo.get_connections_from_env().user_db
mongo_collection = user_db.get_collection(args.mongo_collection)
selected_users = mongo_collection.find(mongo_filter, {'emailsSent': 1})
treated_users = 0
# TODO(cyrille): Make sure errors are logged to sentry.
# TODO(cyrille): If it fails on a specific user, keep going.
for user in selected_users:
emails_sent = user.get('emailsSent', [])
updated_emails_sent = [
_update_email_sent_status(email,
yesterday,
campaign_ids=args.campaigns)
for email in emails_sent
]
mongo_collection.update_one(
{'_id': user['_id']},
{'$set': {
'emailsSent': updated_emails_sent
}})
treated_users += 1
if not treated_users % 100:
logging.info('Treated %d users', treated_users)
def set_date_now(time_proto: timestamp_pb2.Timestamp,
now_value: Optional[datetime.datetime] = None) -> None:
"""Set date with the value from now, without too much precision."""
time_proto.FromDatetime(now_value or now.get())
time_proto.nanos = 0
def _date_from_today(absolute_date: str, num_days_ago: Optional[int]) -> str:
if num_days_ago is None:
return absolute_date
return (now.get() -
datetime.timedelta(days=num_days_ago)).strftime('%Y-%m-%d')
