def fetch_use_case_pools():
"""Retrieve a list of the available pools of anonymized user examples."""
use_case_pools = use_case_pb2.UseCasePools()
use_case_pool_dicts = flask.current_app.config[
'DATABASE'].use_case.aggregate([
{
'$group': {
'_id': '$poolName',
'useCaseCount': {
'$sum': 1
},
'evaluatedUseCaseCount': {
'$sum': {
'$cond': [{
'$gt': ['$evaluation', None]
}, 1, 0]
}
},
'lastUserRegisteredAt': {
'$max': '$userData.registeredAt'
},
}
},
{
'$sort': {
'lastUserRegisteredAt': -1
}
},
])
for use_case_pool_dict in use_case_pool_dicts:
use_case_pool_proto = use_case_pools.use_case_pools.add()
use_case_pool_proto.name = use_case_pool_dict.pop('_id')
proto.parse_from_mongo(use_case_pool_dict, use_case_pool_proto)
return use_case_pools
def main(user_db, dry_run=True):
"""Fix projects with very old field values."""
if dry_run:
print('Running in dry mode, no changes will be pushed to MongoDB.')
# TODO(pascal): Also run this fix on other projects with old fields:
# - the ones with kind == FIND_JOB
# - the ones with no jobSearchStartedAt nor jobSearchHasNotStarted
# - the ones with actionsGeneratedAt
users_to_fix = user_db.find({
'projects.diplomaFulfillmentEstimate': {'$exists': True},
'projects.trainingFulfillmentEstimate': {'$exists': False},
})
user_count = 0
for user_dict in users_to_fix:
user_id = user_dict.pop('_id')
user = user_pb2.User()
proto.parse_from_mongo(user_dict, user)
updated = False
for project in user.projects:
if not fix_project(project, user.profile):
continue
updated = True
if dry_run:
print('Would change project for', user_id, project.project_id)
else:
user_db.update_one(
{'_id': user_id, 'projects.projectId': project.project_id},
{'$set': {'projects.$': json_format.MessageToDict(project)}},
)
if updated:
user_count += 1
print('{} users updated'.format(user_count))
def _get_seasonal_departements(self, project):
"""Compute departements that propose seasonal jobs."""
top_departements = seasonal_jobbing_pb2.MonthlySeasonalJobbingStats()
# TODO(guillaume): Cache this to increase speed.
proto.parse_from_mongo(
project.database.seasonal_jobbing.find_one(
{'_id': project.now.month}), top_departements)
for departement in top_departements.departement_stats:
# TODO(guillaume): If we don't use deeper jobgroups by october 1st 2017, trim the db.
del departement.job_groups[6:]
try:
departement.departement_in_name = geo.get_in_a_departement_text(
departement.departement_id)
except KeyError:
logging.exception(
'Prefix or name not found for departement: %s',
departement.departement_id)
continue
for i, departement in enumerate(
top_departements.departement_stats[::-1]):
if not departement.departement_in_name:
del top_departements.departement_stats[i]
return top_departements or []
def send_reset_password_token(self, email):
"""Sends an email to user with a reset token so that they can reset their password."""
user_dict = self._db.user.find_one({'profile.email': email})
if not user_dict:
flask.abort(403, "Nous n'avons pas d'utilisateur avec cet email : %s" % email)
user_auth_dict = self._db.user_auth.find_one({'_id': user_dict['_id']})
if not user_auth_dict or not user_auth_dict.get('hashedPassword'):
flask.abort(
403, 'Utilisez Facebook ou Google pour vous connecter, comme la première fois.')
hashed_old_password = user_auth_dict.get('hashedPassword')
auth_token = _timestamped_hash(
int(time.time()), email + str(user_dict['_id']) + hashed_old_password)
user_profile = user_pb2.UserProfile()
proto.parse_from_mongo(user_dict.get('profile'), user_profile)
reset_link = parse.urljoin(flask.request.url, '/?' + parse.urlencode({
'email': email,
'resetToken': auth_token}))
template_vars = {
'resetLink': reset_link,
'firstName': user_profile.name,
}
result = mail.send_template(
'71254', user_profile, template_vars, monitoring_category='reset_password')
if result.status_code != 200:
logging.error('Failed to send an email with MailJet:\n %s', result.text)
flask.abort(result.status_code)
def select_advice_for_email(user, weekday, database):
"""Select an advice to promote in a follow-up email."""
if not user.projects:
return None
project = user.projects[0]
if not project.advices:
return None
easy_advice_modules = _easy_advice_modules(database)
history = advisor_pb2.EmailHistory()
history_dict = database.email_history.find_one(
{'_id': objectid.ObjectId(user.user_id)})
proto.parse_from_mongo(history_dict, history)
today = now.get()
last_monday = today - datetime.timedelta(days=today.weekday())
def _score_advice(priority_and_advice):
"""Score piece of advice to compare to others, the higher the better."""
priority, advice = priority_and_advice
score = 0
# Enforce priority advice on Mondays (between -10 and +10).
if weekday == user_pb2.MONDAY:
priority_score = (advice.score
or 5) - 10 * priority / len(project.advices)
score += priority_score
# Enforce easy advice on Fridays (between +0 and +1).
if weekday == user_pb2.FRIDAY and advice.advice_id in easy_advice_modules:
score += 1
random_factor = (advice.score or 5) / 10
last_sent = history.advice_modules[advice.advice_id].ToDatetime()
last_sent_monday = last_sent - datetime.timedelta(
days=last_sent.weekday())
# Do not send the same advice in the same week (+0 or -20).
if last_sent_monday >= last_monday:
score -= 20
# Reduce the random boost if advice was sent in past weeks (*0.2 the
# week just before, *0.45 the week before that, *0.585, *0.669, …) .
else:
num_weeks_since_last_sent = (last_monday -
last_sent_monday).days / 7
random_factor *= .2**(1 / num_weeks_since_last_sent)
# Randomize pieces of advice with the same score (between +0 and +1).
score += random.random() * random_factor
return score
candidates = sorted(enumerate(project.advices),
key=_score_advice,
reverse=True)
return next(advice for priority, advice in candidates)
def select_tips_for_email(user,
project,
piece_of_advice,
database,
num_tips=3):
"""Select tips to promote an advice in a follow-up email."""
all_templates = list_all_tips(user,
project,
piece_of_advice,
database,
filter_tip=lambda t: t.is_ready_for_email)
# TODO(pascal): Factorize with above.
history = advisor_pb2.EmailHistory()
history_dict = database.email_history.find_one(
{'_id': objectid.ObjectId(user.user_id)})
proto.parse_from_mongo(history_dict, history)
today = now.get()
def _score_tip_template(tip_template):
"""Score tip template to compare to others, the higher the better."""
score = 0
last_sent = history.tips[tip_template.action_template_id].ToDatetime()
# Score higher the tips that have not been sent for longer time.
score += (today - last_sent).days
# Randomize tips with the same score.
score += random.random()
return score
selected_templates = sorted(all_templates,
key=_score_tip_template)[-num_tips:]
if not selected_templates:
return None
# Instantiate actual tips.
selected_tips = [
action.instantiate(action_pb2.Action(),
user,
project,
template,
set(),
database,
None,
for_email=True) for template in selected_templates
]
# Replicate tips if we do not have enough.
while len(selected_tips) < num_tips:
selected_tips.extend(selected_tips)
return selected_tips[:num_tips]
def fetch_use_cases(pool_name):
"""Retrieve a list of anonymized user examples from one pool."""
use_cases = use_case_pb2.UseCases()
use_case_dicts = flask.current_app.config['DATABASE'].use_case\
.find({'poolName': pool_name}).sort('indexInPool', 1)
for use_case_dict in use_case_dicts:
use_case_proto = use_cases.use_cases.add()
use_case_proto.use_case_id = use_case_dict.pop('_id')
proto.parse_from_mongo(use_case_dict, use_case_proto)
return use_cases
def _google_authenticate(self, token_id):
try:
id_info = client.verify_id_token(token_id, GOOGLE_SSO_CLIENT_ID)
except crypt.AppIdentityError as error:
flask.abort(401, "Mauvais jeton d'authentification : %s" % error)
if id_info.get('iss') not in _GOOGLE_SSO_ISSUERS:
flask.abort(
401, "Fournisseur d'authentification invalide : %s." %
id_info.get('iss', '<none>'))
response = user_pb2.AuthResponse()
user_dict = self._db.user.find_one({'googleId': id_info['sub']})
user_id = str(user_dict['_id']) if user_dict else ''
if proto.parse_from_mongo(user_dict, response.authenticated_user):
response.authenticated_user.user_id = user_id
else:
self._assert_user_not_existing(id_info['email'])
response.authenticated_user.profile.email = id_info['email']
response.authenticated_user.profile.picture_url = id_info.get(
'picture', '')
response.authenticated_user.google_id = id_info['sub']
self._save_new_user(response.authenticated_user)
response.is_new_user = True
return response
def _get_user_data(user_id):
"""Load user data from DB."""
user_dict = _DB.user.find_one({'_id': _safe_object_id(user_id)})
user_proto = user_pb2.User()
if not proto.parse_from_mongo(user_dict, user_proto):
# Switch to raising an error if you move this function in a lib.
flask.abort(404, 'Utilisateur "%s" inconnu.' % user_id)
_populate_feature_flags(user_proto)
for project in user_proto.projects:
# TODO(pascal): Update existing users and get rid of diploma_fulfillment_estimate.
if not project.training_fulfillment_estimate and project.diploma_fulfillment_estimate:
project.training_fulfillment_estimate = TRAINING_ESTIMATION.get(
project.diploma_fulfillment_estimate,
project_pb2.UNKNOWN_TRAINING_FULFILLMENT)
# TODO(pascal): Update existing users and get rid of FIND_JOB.
if project.kind == project_pb2.FIND_JOB:
if user_proto.profile.situation == user_pb2.LOST_QUIT:
project.kind = project_pb2.FIND_A_NEW_JOB
elif user_proto.profile.situation == user_pb2.FIRST_TIME:
project.kind = project_pb2.FIND_A_FIRST_JOB
else:
project.kind = project_pb2.FIND_ANOTHER_JOB
project.ClearField('diploma_fulfillment_estimate')
# TODO(pascal): Remove the fields completely after this has been live for a
# week.
user_proto.profile.ClearField('city')
user_proto.profile.ClearField('latest_job')
user_proto.profile.ClearField('situation')
return user_proto
def _facebook_authenticate(self, signed_request):
try:
[encoded_signature, payload] = signed_request.split('.')
data = json.loads(_base64_url_decode(payload).decode('utf-8'))
actual_signature = _base64_url_decode(encoded_signature)
except ValueError as error:
flask.abort(422, error)
for required_field in ('algorithm', 'user_id'):
if not data.get(required_field):
flask.abort(
422,
'Le champs %s est requis : %s' % (required_field, data))
if data['algorithm'].lower() != 'hmac-sha256':
flask.abort(
422,
'Algorithme d\'encryption inconnu "%s"' % data['algorithm'])
expected_signature = hmac.new(_FACEBOOK_SECRET,
payload.encode('utf-8'),
hashlib.sha256).digest()
if expected_signature != actual_signature:
flask.abort(403, 'Mauvaise signature')
response = user_pb2.AuthResponse()
user_dict = self._db.user.find_one({'facebookId': data['user_id']})
user_id = str(user_dict['_id']) if user_dict else ''
if proto.parse_from_mongo(user_dict, response.authenticated_user):
response.authenticated_user.user_id = user_id
else:
response.authenticated_user.facebook_id = data['user_id']
self._save_new_user(response.authenticated_user)
response.is_new_user = True
return response
def main():
"""Send an email to a user to tell them to focus on the network."""
campaign_id = 'focus-network'
template_id = '205970'
email_count = 0
selected_users = _DB.user.find({
'registeredAt': {
'$gt': '2017-04-01',
'$lt': '2017-07-10',
},
'projects.networkEstimate': 1,
})
for user_dict in selected_users:
user_id = user_dict.pop('_id')
user = user_pb2.User()
proto.parse_from_mongo(user_dict, user)
if any(email.campaign_id == campaign_id for email in user.emails_sent):
# We already sent the email to that person.
continue
template_vars = network_vars(user)
if not template_vars:
continue
req = mail.send_template(template_id, user.profile, template_vars)
print('Email sent to %s' % user.profile.email)
if req.status_code != 200:
logging.warning('Error while sending an email: %d',
req.status_code)
continue
email_sent = user.emails_sent.add()
email_sent.sent_at.GetCurrentTime()
email_sent.mailjet_template = template_id
email_sent.campaign_id = campaign_id
_DB.user.update_one({'_id': user_id}, {
'$set': {
'emailsSent':
json_format.MessageToDict(user).get('emailsSent', []),
}
})
email_count += 1
return email_count
def get_dashboard_export(dashboard_export_id):
"""Retrieve an export of the user's current dashboard."""
dashboard_export_json = _DB.dashboard_exports.find_one(
{'_id': _safe_object_id(dashboard_export_id)})
dashboard_export = export_pb2.DashboardExport()
if not proto.parse_from_mongo(dashboard_export_json, dashboard_export):
flask.abort(404, 'Export "%s" introuvable.' % dashboard_export_id)
dashboard_export.dashboard_export_id = dashboard_export_id
return dashboard_export
def test_unknown_field(self):
"""Unknown fields do not make the function choke."""
job_group = job_pb2.JobGroup()
self.assertTrue(
proto.parse_from_mongo({
'romeId': 'A123',
'unknownField': 14
}, job_group))
self.assertEqual('A123', job_group.rome_id)
def get_usage_stats():
"""Get stats of the app usage."""
now_utc = now.get().astimezone(datetime.timezone.utc)
start_of_second = now_utc.replace(microsecond=0, tzinfo=None)
last_week = start_of_second - datetime.timedelta(days=7)
yesterday = start_of_second - datetime.timedelta(days=1)
# Compute daily scores count.
daily_scores_count = collections.defaultdict(int)
last_day_users = _DB.user.find(
{
'registeredAt': {
'$gt': _datetime_to_json_string(yesterday),
'$lte': _datetime_to_json_string(start_of_second),
},
},
{
'profile.email': 1,
'projects': 1,
'registeredAt': 1,
},
)
for user_dict in last_day_users:
user_proto = user_pb2.User()
proto.parse_from_mongo(user_dict, user_proto)
if _is_test_user(user_proto):
continue
for project in user_proto.projects:
if project.feedback.score:
daily_scores_count[project.feedback.score] += 1
# Compute weekly user count.
weekly_new_user_count = _DB.user.find({
'registeredAt': {
'$gt': _datetime_to_json_string(last_week),
'$lte': _datetime_to_json_string(start_of_second),
}
}).count()
return stats_pb2.UsersCount(
total_user_count=_DB.user.count(),
weekly_new_user_count=weekly_new_user_count,
daily_scores_count=daily_scores_count,
)
def main(user_db, dry_run=True):
"""Fix projects with very old field values."""
if dry_run:
print('Running in dry mode, no changes will be pushed to MongoDB.')
users_to_fix = user_db.find({
'employment_status': {
'$elemMatch': {
'createdAt': {
'$lt': '2017-10-12'
},
'seeking': {
'$exists': False
}
}
},
})
user_count = 0
for user_dict in users_to_fix:
user_id = user_dict.pop('_id')
user = user_pb2.User()
proto.parse_from_mongo(user_dict, user)
updated = False
for pos, status in enumerate(user.employment_status):
if status.seeking:
continue
updated = True
if dry_run:
print('Would change status for', user_id, pos)
else:
status.seeking = user_pb2.STOP_SEEKING
user_db.update_one(
{'_id': user_id},
{
'$set': {
'employment_status.{}'.format(pos):
json_format.MessageToDict(status)
}
},
)
if updated:
user_count += 1
print('{} users updated'.format(user_count))
def _update_email_sent_status(email_sent_dict, email_address):
email_sent = user_pb2.EmailSent()
proto.parse_from_mongo(email_sent_dict, email_sent)
if email_sent.status != user_pb2.EMAIL_SENT_UNKNOWN:
# TODO(pascal): Check the status again if too old.
return email_sent_dict
message = _find_message(email_sent, email_address)
if not message:
logging.warning('Could not find a message in MailJet.')
return email_sent_dict
email_sent.mailjet_message_id = message.get('ID',
email_sent.mailjet_message_id)
email_sent.last_status_checked_at.GetCurrentTime()
email_sent.last_status_checked_at.nanos = 0
email_sent.status = user_pb2.EmailSentStatus.Value('EMAIL_SENT_{}'.format(
message.get('Status', 'unknown').upper()))
return json_format.MessageToDict(email_sent)
def list_nearby_cities(self, project):
"""Compute and store all interesting cities that are not too close and not too far.
Those cities will be used by the Commute advice.
"""
job_group = project.details.target_job.job_group.rome_id
all_cities = commute_pb2.HiringCities()
proto.parse_from_mongo(
project.database.hiring_cities.find_one({'_id': job_group}),
all_cities)
interesting_cities_for_rome = all_cities.hiring_cities
if not interesting_cities_for_rome:
return []
target_city = geo_pb2.FrenchCity()
mongo_city = project.database.cities.find_one(
{'_id': project.details.mobility.city.city_id})
if not mongo_city:
return []
proto.parse_from_mongo(mongo_city, target_city)
commuting_cities = list(
_get_commuting_cities(interesting_cities_for_rome, target_city))
obvious_cities = [
city for city in commuting_cities
if city.distance_km < _MIN_CITY_DISTANCE
]
interesting_cities = [
city for city in commuting_cities
if city.distance_km >= _MIN_CITY_DISTANCE
]
# If there is only one city nearby and no obvious city, the nearby city becomes obvious, so
# we do not recommend it.
if len(interesting_cities) == 1 and not obvious_cities:
return []
return interesting_cities
def load_set(cls, filename):
"""Load a set of personas from a JSON file."""
with open(filename) as personas_file:
personas_json = json.load(personas_file)
personas = {}
for name, blob in personas_json.items():
user_profile = user_pb2.UserProfile()
assert proto.parse_from_mongo(blob['user'], user_profile)
features_enabled = user_pb2.Features()
if 'featuresEnabled' in blob:
assert proto.parse_from_mongo(blob['featuresEnabled'],
features_enabled)
project = project_pb2.Project()
assert proto.parse_from_mongo(blob['project'], project)
assert name not in personas
personas[name] = cls(name,
user_profile=user_profile,
project=project,
features_enabled=features_enabled)
return personas
def user_to_use_case(user, pool_name, index_in_pool):
"""Extracts a use case from a real user."""
use_case = use_case_pb2.UseCase()
if not proto.parse_from_mongo(user, use_case.user_data):
return None
use_case.title = next((p.title for p in use_case.user_data.projects), '')
anonymize_proto(use_case.user_data, field_usages_to_clear={
options_pb2.PERSONAL_IDENTIFIER, options_pb2.APP_ONLY, options_pb2.ALGORITHM_RESULT,
})
use_case.pool_name = pool_name
use_case.index_in_pool = index_in_pool
use_case.use_case_id = '{}_{:02x}'.format(pool_name, index_in_pool)
return use_case
def test_weird_objects(self, mock_warning):
"""Raises a TypeError when an object is not of the right type."""
job_group = job_pb2.JobGroup()
self.assertFalse(proto.parse_from_mongo({'romeId': 123}, job_group))
mock_warning.assert_called_once()
self.assertEqual(
'Error %s while parsing a JSON dict for proto type %s:\n%s',
mock_warning.call_args[0][0])
self.assertEqual(
'Failed to parse romeId field: expected string or bytes-like object.',
str(mock_warning.call_args[0][1]))
self.assertEqual('JobGroup', str(mock_warning.call_args[0][2]))
self.assertEqual("{'romeId': 123}", str(mock_warning.call_args[0][3]))
def volunteering_missions(self, project):
"""Return a list of volunteering mission close to the project."""
departement_id = project.details.mobility.city.departement_id
# Get data from MongoDB.
volunteering_missions_dict = collections.defaultdict(
association_pb2.VolunteeringMissions)
collection = project.database.volunteering_missions
for record in collection.find({'_id': {'$in': [departement_id, '']}}):
record_id = record.pop('_id')
proto.parse_from_mongo(record,
volunteering_missions_dict[record_id])
# TODO(pascal): First get missions from target city if any.
# Merge data.
project_missions = association_pb2.VolunteeringMissions()
for scope in [departement_id, '']:
for mission in volunteering_missions_dict[scope].missions:
mission.is_available_everywhere = not scope
project_missions.missions.add().CopyFrom(mission)
return project_missions
def _get_user_data(user_id):
"""Load user data from DB."""
user_dict = _DB.user.find_one({'_id': _safe_object_id(user_id)})
user_proto = user_pb2.User()
if not proto.parse_from_mongo(user_dict, user_proto):
# Switch to raising an error if you move this function in a lib.
flask.abort(404, 'Utilisateur "%s" inconnu.' % user_id)
_populate_feature_flags(user_proto)
for project in user_proto.projects:
# TODO(pascal): Update existing users and get rid of diploma_fulfillment_estimate.
if not project.training_fulfillment_estimate and project.diploma_fulfillment_estimate:
project.training_fulfillment_estimate = TRAINING_ESTIMATION.get(
project.diploma_fulfillment_estimate,
project_pb2.UNKNOWN_TRAINING_FULFILLMENT)
# TODO(pascal): Update existing users and get rid of FIND_JOB.
if project.kind == project_pb2.FIND_JOB:
if user_proto.profile.situation == user_pb2.LOST_QUIT:
project.kind = project_pb2.FIND_A_NEW_JOB
elif user_proto.profile.situation == user_pb2.FIRST_TIME:
project.kind = project_pb2.FIND_A_FIRST_JOB
else:
project.kind = project_pb2.FIND_ANOTHER_JOB
# TODO(pascal): Update existing users and get rid of job_search_length_months.
if not (project.job_search_started_at.seconds or project.job_search_has_not_started) \
and project.job_search_length_months:
if project.job_search_length_months < 0:
project.job_search_has_not_started = True
else:
job_search_length_days = 30.5 * project.job_search_length_months
job_search_length_duration = datetime.timedelta(
days=job_search_length_days)
project.job_search_started_at.FromDatetime(
project.created_at.ToDatetime() -
job_search_length_duration)
project.job_search_started_at.nanos = 0
project.ClearField('diploma_fulfillment_estimate')
project.ClearField('actions_generated_at')
# TODO(pascal): Remove the fields completely after this has been live for a
# week.
user_proto.profile.ClearField('city')
user_proto.profile.ClearField('latest_job')
user_proto.profile.ClearField('situation')
return user_proto
def _populate_job_stats_dict(local_job_stats, city):
local_stats_ids = dict(
('%s:%s' % (city.departement_id, job_group_id), job_group_id)
for job_group_id in local_job_stats)
# Import most stats from local_diagnosis.
local_stats = _DB.local_diagnosis.find(
{'_id': {
'$in': list(local_stats_ids)
}})
for job_group_local_stats in local_stats:
job_group_id = local_stats_ids[job_group_local_stats['_id']]
proto.parse_from_mongo(job_group_local_stats,
local_job_stats[job_group_id])
# Enrich with the # of job offers.
job_group_offers_counts = _DB.recent_job_offers.find(
{'_id': {
'$in': list(local_stats_ids)
}})
for job_group_offers_count in job_group_offers_counts:
job_group_id = local_stats_ids[job_group_offers_count['_id']]
proto.parse_from_mongo(job_group_offers_count,
local_job_stats[job_group_id])
def main(database, base_url, now):
"""Send an email with a selected advice to a list of users."""
# Week day as a user_pb2.WeekDay value.
weekday = now.weekday() + 1
query = {
'profile.emailDays': user_pb2.WeekDay.Name(weekday),
'projects.advices.score': {'$gt': 0},
'featuresEnabled.advisorEmail': 'ACTIVE',
}
count = 0
cool_down_time_beginning = now - _COOL_DOWN_TIME
user_iterator = database.user.find(query)
errors = []
for user_in_db in _break_on_signal([signal.SIGTERM], user_iterator):
user = user_pb2.User()
user_id = user_in_db['_id']
if not proto.parse_from_mongo(user_in_db, user):
# Skip silently (the proto library already logs the error).
continue
if user.last_email_sent_at.ToDatetime() > cool_down_time_beginning:
# Skip silently.
continue
if _deactivate_if_never_opened(user_id, user, database.user):
continue
try:
result = send_email_to_user(user, base_url, weekday, database)
except (IOError, json_format.ParseError) as err:
errors.append('%s - %s' % (err, user_id))
logging.error(err)
continue
if not result:
continue
if not DRY_RUN:
database.user.update_one(
{'_id': user_id},
{'$set': {'lastEmailSentAt': now}})
count += 1
_send_reports(count, errors, weekday)
def _google_authenticate(self, token_id):
id_info = decode_google_id_token(token_id)
response = user_pb2.AuthResponse()
user_dict = self._db.user.find_one({'googleId': id_info['sub']})
user_id = str(user_dict['_id']) if user_dict else ''
if proto.parse_from_mongo(user_dict, response.authenticated_user):
response.authenticated_user.user_id = user_id
else:
self._assert_user_not_existing(id_info['email'])
response.authenticated_user.profile.email = id_info['email']
response.authenticated_user.profile.picture_url = id_info.get('picture', '')
response.authenticated_user.google_id = id_info['sub']
self._save_new_user(response.authenticated_user)
response.is_new_user = True
# TODO(benoit): Check token provide enough guarantee for this use case.
response.auth_token = create_token(response.authenticated_user.user_id, 'auth')
return response
def redirect_eterritoire(city_id):
"""Redirect to the e-Territoire page for a city."""
link = association_pb2.SimpleLink()
proto.parse_from_mongo(_DB.eterritoire_links.find_one({'_id': city_id}),
link)
return flask.redirect('http://www.eterritoire.fr%s' % link.path)
def blast_campaign(campaign_id,
action,
registered_from,
registered_to,
dry_run_email='',
user_hash=''):
"""Send a campaign of personalized emails."""
if action == 'send' and auth.SECRET_SALT == auth.FAKE_SECRET_SALT:
raise ValueError('Set the prod SECRET_SALT env var before continuing.')
campaign = _CAMPAIGNS[campaign_id]
template_id = campaign.mailjet_template
selected_users = _DB.user.find(
dict(
campaign.mongo_filters, **{
'profile.email': {
'$not': re.compile(r'@example.com$')
},
'registeredAt': {
'$gt': registered_from,
'$lt': registered_to,
}
}))
email_count = 0
email_errors = 0
for user_dict in selected_users:
user_id = user_dict.pop('_id')
user = user_pb2.User()
proto.parse_from_mongo(user_dict, user)
user.user_id = str(user_id)
if user_hash and not user.user_id.startswith(user_hash):
continue
if any(email.campaign_id == campaign_id for email in user.emails_sent):
# We already sent the email to that person.
continue
template_vars = campaign.get_vars(user)
if not template_vars:
continue
if action == 'list':
logging.info('%s %s', user.user_id, user.profile.email)
continue
if action == 'dry-run':
user.profile.email = dry_run_email
if action in ('dry-run', 'send'):
res = mail.send_template(template_id,
user.profile,
template_vars,
sender_email=campaign.sender_email,
sender_name=campaign.sender_email)
logging.info('Email sent to %s', user.profile.email)
if action == 'dry-run':
try:
res.raise_for_status()
except requests.exceptions.HTTPError:
raise ValueError(
'Could not send email for vars:\n{}'.format(template_vars))
elif res.status_code != 200:
logging.warning('Error while sending an email: %d',
res.status_code)
email_errors += 1
continue
sent_response = res.json()
message_id = next(iter(sent_response.get('Sent', [])),
{}).get('MessageID', 0)
if not message_id:
logging.warning('Impossible to retrieve the sent email ID:\n%s',
sent_response)
if action == 'dry-run':
return 1
email_sent = user.emails_sent.add()
email_sent.sent_at.GetCurrentTime()
email_sent.sent_at.nanos = 0
email_sent.mailjet_template = template_id
email_sent.campaign_id = campaign_id
email_sent.mailjet_message_id = message_id
_DB.user.update_one({'_id': user_id}, {
'$set': {
'emailsSent':
json_format.MessageToDict(user).get('emailsSent', []),
}
})
email_count += 1
if email_count % 100 == 0:
print('{} emails sent ...'.format(email_count))
if action == 'send':
report.notify_slack(
"Report for 3 month employment-status blast: I've sent {:d} emails (and got {:d} \
errors).".format(email_count, email_errors))
return email_count
def _password_authenticate(self, auth_request):
now = int(time.time())
response = user_pb2.AuthResponse()
response.hash_salt = _timestamped_hash(now, auth_request.email)
user_dict = self._db.user.find_one(
{'profile.email': auth_request.email})
if not user_dict:
return self._password_register(auth_request, response)
user_id = str(user_dict['_id'])
user_auth_dict = self._db.user_auth.find_one({'_id': user_dict['_id']})
if not user_auth_dict:
if user_dict.get('googleId'):
auth_method = ' (Google)'
elif user_dict.get('facebookId'):
auth_method = ' (Facebook)'
else:
auth_method = ''
flask.abort(
403,
"L'utilisateur existe mais utilise un autre moyen de connexion%s."
% auth_method)
if not auth_request.hashed_password:
# User exists but did not sent a passwordt: probably just getting some fresh salt.
return response
if auth_request.auth_token:
self._reset_password(auth_request, user_id, user_auth_dict)
user_dict['userId'] = user_id
if not proto.parse_from_mongo(user_dict,
response.authenticated_user):
flask.abort(
500,
'Les données utilisateur sont corrompues dans la base de données.'
)
return response
if not auth_request.hash_salt:
# User exists but has not sent salt: probably just
# getting some fresh salt.
return response
# Check that salt is valid.
salt = auth_request.hash_salt
try:
if not _assert_valid_salt(salt, auth_request.email, now):
return response
except ValueError as error:
flask.abort(
403, "Le sel n'a pas été généré par ce serveur : %s." % error)
stored_hashed_password = user_auth_dict.get('hashedPassword')
hashed_password = hashlib.sha1()
hashed_password.update(salt.encode('ascii'))
hashed_password.update(stored_hashed_password.encode('ascii'))
request_hashed_password = binascii.unhexlify(
auth_request.hashed_password)
if request_hashed_password != hashed_password.digest():
flask.abort(403, 'Mot de passe erroné.')
if not proto.parse_from_mongo(user_dict, response.authenticated_user):
flask.abort(
500,
'Les données utilisateur sont corrompues dans la base de données.'
)
response.authenticated_user.user_id = user_id
return response
def test_timestamp(self):
"""Parse correctly Python timestamps."""
action = action_pb2.Action()
now = datetime.datetime.now()
self.assertTrue(proto.parse_from_mongo({'createdAt': now}, action))
self.assertEqual(now, action.created_at.ToDatetime())
