def test_force_recreate(self) -> None: """ES drops index and recreates it at start.""" called_indices_methods: list[str] = [] def _save_called_func(method_name: str) -> Callable[[str], None]: def _side_effect(index: str) -> None: # pylint: disable=unused-argument called_indices_methods.append(method_name) return None return _side_effect self.mock_elasticsearch.indices.exists.return_value = True self.mock_elasticsearch.indices.delete.side_effect = _save_called_func( 'delete') self.mock_elasticsearch.indices.create.side_effect = _save_called_func( 'create') sync_user_elasticsearch.main(self.mock_elasticsearch, [ '--registered-from', '2017-07', '--index', 'bobusers', '--force-recreate', '--no-dry-run', '--disable-sentry' ]) self.mock_elasticsearch.indices.delete.assert_called_once_with( index='bobusers') self.mock_elasticsearch.indices.create.assert_called_once_with( index='bobusers') self.assertEqual(['delete', 'create'], called_indices_methods)
def _check_city_corner_case(self, expected_city: dict[str, Any], city_json: dict[str, Any]) -> None: self._user_db.user.insert_one({ 'projects': [{ 'city': { 'cityId': '69123' } }], 'registeredAt': '2017-07-15T18:06:08Z', }) self._db.cities.insert_one(city_json | {'_id': '69123'}) sync_user_elasticsearch.main(self.mock_elasticsearch, [ '--registered-from', '2017-07', '--no-dry-run', '--disable-sentry' ]) # TODO(cyrille): DRY with _compute_user_data. self.mock_elasticsearch.update.assert_called_once() body = self.mock_elasticsearch.update.call_args[1]['body']['doc'] city = body['project']['city'] del city['regionName'] del city['urbanScore'] self.assertEqual(expected_city, body['project']['city'])
def _compute_user_data(self, user: user_pb2.User) -> Dict[str, Any]: if not user.HasField('registered_at'): user.registered_at.GetCurrentTime() self._user_db.user.drop() self._user_db.user.insert_one(json_format.MessageToDict(user)) sync_user_elasticsearch.main(self.mock_elasticsearch, [ '--registered-from', '2017-07', '--no-dry-run', '--disable-sentry' ]) kwargs = self.mock_elasticsearch.create.call_args[1] return typing.cast(Dict[str, Any], json.loads(kwargs.pop('body')))
def test_update_index(self) -> None: """No ES index operation if index already exists.""" self.mock_elasticsearch.indices.exists.return_value = True sync_user_elasticsearch.main(self.mock_elasticsearch, [ '--registered-from', '2017-07', '--index', 'bobusers', '--no-dry-run', '--disable-sentry' ]) self.assertFalse(self.mock_elasticsearch.indices.delete.called) self.assertFalse(self.mock_elasticsearch.indices.create.called)
def test_create_index(self) -> None: """Create ES index if it doesn't already exist.""" self.mock_elasticsearch.indices.exists.return_value = False sync_user_elasticsearch.main(self.mock_elasticsearch, [ '--registered-from', '2017-07', '--index', 'bobusers', '--no-dry-run', '--disable-sentry' ]) self.assertFalse(self.mock_elasticsearch.indices.delete.called) self.mock_elasticsearch.indices.create.assert_called_once_with( index='bobusers')
def test_report_sentry(self, mock_error: mock.MagicMock) \ -> None: """Test the error message if we forgot to set SENTRY reporting.""" sync_user_elasticsearch.main( self.mock_elasticsearch, ['--registered-from', '2017-07', '--no-dry-run']) mock_error.assert_called_once_with( 'Please set SENTRY_DSN to enable logging to Sentry, or use --disable-sentry option' )
def test_ffs(self) -> None: """Info from the FFS.""" self._user_db.user.insert_one({ 'registeredAt': '2022-02-12T18:06:08Z', 'emailsSent': [ { 'campaignId': 'first-followup-survey', # 7 days after registration. 'sentAt': '2022-02-19T18:06:08Z', 'status': 'EMAIL_SENT_SENT', }, { 'campaignId': 'focus-spontaneous', 'sentAt': '2022-02-21T18:06:08Z', 'status': 'EMAIL_SENT_OPENED', }, ], 'firstFollowupSurveyResponse': { # 8 days after registration. 'respondedAt': '2022-02-20T19:06:00Z', 'hasTriedSomethingNew': True, 'newIdeasScore': 4, 'improveSelfConfidenceScore': 2, }, }) sync_user_elasticsearch.main(self.mock_elasticsearch, [ '--registered-from', '2022-02-01', '--no-dry-run', '--disable-sentry' ]) self.mock_elasticsearch.update.assert_called_once() kwargs = self.mock_elasticsearch.update.call_args[1] self.assertIn('body', kwargs) self.assertIn('doc', kwargs['body']) doc = kwargs['body']['doc'] self.assertTrue(json.dumps(doc), msg='The document should be serializable') self.assertLessEqual({'ffsRequest', 'ffsResponse'}, doc.keys()) self.assertEqual({ 'hasResponded': True, 'sentAfterDays': 7 }, doc['ffsRequest']) self.assertEqual( { 'hasTriedSomethingNew': True, 'improveSelfConfidenceScore': 2, 'newIdeasScore': 4, 'respondedDaysAfterRegistration': 8, }, doc['ffsResponse'])
def _test_project(self, project_json: dict[str, Any]) -> dict[str, Any]: self._user_db.user.insert_one({ 'registeredAt': '2022-02-12T18:06:08Z', 'projects': [project_json], }) sync_user_elasticsearch.main(self.mock_elasticsearch, [ '--registered-from', '2022-02-01', '--no-dry-run', '--disable-sentry' ]) self.mock_elasticsearch.update.assert_called_once() kwargs = self.mock_elasticsearch.update.call_args[1] self.assertIn('body', kwargs) self.assertIn('doc', kwargs['body']) doc = kwargs['body']['doc'] self.assertTrue(json.dumps(doc), msg='The document should be serializable') self.assertIn('project', doc) return typing.cast(dict[str, Any], doc['project'])
def _check_city_corner_case(self, expected_city: Dict[str, Any], city_json: Dict[str, Any]) -> None: self._user_db.user.insert_one({ 'projects': [{ 'city': { 'cityId': '69123' } }], 'registeredAt': '2017-07-15T18:06:08Z', }) self._db.cities.insert_one(dict(**city_json, _id='69123')) sync_user_elasticsearch.main(self.mock_elasticsearch, [ '--registered-from', '2017-07', '--no-dry-run', '--disable-sentry' ]) self.mock_elasticsearch.create.assert_called_once() body = json.loads(self.mock_elasticsearch.create.call_args[1]['body']) city = body['project']['city'] del city['regionName'] del city['urbanScore'] self.assertEqual(expected_city, body['project']['city'])
def test_main(self, mock_elasticsearch): """Test main.""" self.maxDiff = None # pylint: disable=invalid-name self._db.user.insert_one({ 'profile': { 'email': '*****@*****.**', 'gender': 'MASCULINE', 'yearOfBirth': 1982, 'highestDegree': 'DEA_DESS_MASTER_PHD', 'origin': 'FROM_A_FRIEND', }, 'projects': [{ 'kind': 'FIND_ANOTHER_JOB', 'targetJob': { 'name': 'Boulanger', 'jobGroup': { 'name': 'Boulangerie' }, }, 'mobility': { 'areaType': 'REGION', 'city': { 'cityId': '69123', 'name': 'Lyon', 'departementName': 'Rhône', 'regionName': 'Auvergne-Rhône-Alpes', 'urbanScore': 7, }, }, 'jobSearchLengthMonths': 4, 'advices': [{ 'adviceId': 'network', 'numStars': 3, }], 'feedback': { 'score': 5, }, }], 'employmentStatus': [{ 'bobHasHelped': 'YES', }], 'clientMetrics': { 'amplitudeId': '1234ab34f13e5', 'firstSessionDurationSeconds': 250, }, 'emailsSent': [ # Ignored because there's another one that was read later. { 'campaignId': 'focus-network', 'sentAt': '2017-10-15T18:06:08Z', 'status': 'EMAIL_SENT_SENT', }, { 'campaignId': 'focus-spontaneous', 'sentAt': '2017-10-15T18:06:08Z', 'status': 'EMAIL_SENT_OPENED', }, { 'campaignId': 'focus-network', 'sentAt': '2017-11-15T18:06:08Z', 'status': 'EMAIL_SENT_CLICKED', }, ], 'registeredAt': '2017-07-15T18:06:08Z', }) user_id = str(self._db.user.find_one()['_id']) sync_user_elasticsearch.main( ['--registered-from', '2017-07', '--no-dry-run']) mock_elasticsearch.create.assert_called_once() kwargs = mock_elasticsearch.create.call_args[1] body = kwargs.pop('body') self.assertEqual( { 'index': 'bobusers', 'doc_type': 'user', 'id': user_id, }, kwargs) self.assertEqual( { 'profile': { 'ageGroup': '35-44', 'gender': 'MASCULINE', 'hasHandicap': False, 'highestDegree': 'DEA_DESS_MASTER_PHD', 'origin': 'FROM_A_FRIEND', 'frustrations': [], }, 'project': { 'advices': ['network'], 'job_search_length_months': 4, 'isComplete': True, 'kind': 'FIND_ANOTHER_JOB', 'mobility': { 'areaType': 'REGION', 'city': { 'regionName': 'Auvergne-Rhône-Alpes', 'urbanScore': 7, }, }, 'targetJob': { 'name': 'Boulanger', 'job_group': { 'name': 'Boulangerie', }, }, 'feedbackScore': 5, 'feedbackLoveScore': 1, }, 'registeredAt': '2017-07-15T18:06:08Z', 'employmentStatus': { 'bobHasHelped': 'YES', 'bobHasHelpedScore': 1, }, 'emailsSent': { 'focus-network': 'EMAIL_SENT_CLICKED', 'focus-spontaneous': 'EMAIL_SENT_OPENED', }, 'clientMetrics': { 'firstSessionDurationSeconds': 250, }, }, json.loads(body))
def test_main(self, mock_randint: mock.MagicMock) -> None: """Test main.""" mock_randint.return_value = 42 self.maxDiff = None # pylint: disable=invalid-name self._user_db.user.insert_one({ 'profile': { 'canTutoie': True, 'coachingEmailFrequency': 'EMAIL_MAXIMUM', 'email': '*****@*****.**', 'gender': 'MASCULINE', 'yearOfBirth': 1982, 'highestDegree': 'DEA_DESS_MASTER_PHD', 'origin': 'FROM_A_FRIEND', }, 'projects': [{ 'createdAt': '2018-12-01T00:00:00Z', 'kind': 'FIND_ANOTHER_JOB', 'targetJob': { 'name': 'Boulanger', 'jobGroup': { 'name': 'Boulangerie', 'romeId': 'D0001' }, }, 'areaType': 'REGION', 'city': { 'cityId': '69123', 'name': 'Lyon', 'departementName': 'Rhône', 'regionName': 'Auvergne-Rhône-Alpes', 'urbanScore': 7, }, 'jobSearchStartedAt': '2018-08-01T00:00:00Z', 'minSalary': 45000, 'advices': [ { 'adviceId': 'network', 'numStars': 3, 'status': 'ADVICE_RECOMMENDED', }, { 'adviceId': 'read-more', 'numStars': 1, 'status': 'ADVICE_READ', }, { 'adviceId': 'life-balance', 'numStars': 1, 'status': 'ADVICE_READ', }, ], 'feedback': { 'score': 5, }, 'diagnostic': { 'categoryId': 'stuck-market', }, }], 'employmentStatus': [{ 'bobHasHelped': 'YES', 'bobRelativePersonalization': 12, 'createdAt': '2017-07-25T18:06:08Z', 'otherCoachesUsed': ['PE_COUNSELOR_MEETING', 'MUTUAL_AID_ORGANIZATION'], }], 'clientMetrics': { 'amplitudeId': '1234ab34f13e5', 'firstSessionDurationSeconds': 250, 'isFirstSessionMobile': 'TRUE', }, 'emailsSent': [ # Ignored because there's another one that was read later. { 'campaignId': 'focus-network', 'sentAt': '2017-10-15T18:06:08Z', 'status': 'EMAIL_SENT_SENT', }, { 'campaignId': 'focus-spontaneous', 'sentAt': '2017-10-15T18:06:08Z', 'status': 'EMAIL_SENT_OPENED', }, { 'campaignId': 'focus-network', 'sentAt': '2017-11-15T18:06:08Z', 'status': 'EMAIL_SENT_CLICKED', }, ], 'origin': { 'source': 'facebook', 'medium': 'ad', }, 'registeredAt': '2017-07-15T18:06:08Z', 'hasAccount': True, }) self._db.cities.insert_one({ '_id': '69123', 'name': 'Lyon', 'urbanContext': 2, }) user_id = str(self._user_db.user.find_one()['_id']) sync_user_elasticsearch.main(self.mock_elasticsearch, [ '--registered-from', '2017-07', '--no-dry-run', '--disable-sentry' ]) self.mock_elasticsearch.create.assert_called_once() kwargs = self.mock_elasticsearch.create.call_args[1] body = kwargs.pop('body') self.assertEqual( { 'index': 'bobusers', 'doc_type': '_doc', 'id': user_id, }, kwargs) self.assertEqual( { 'randomGroup': .42, 'profile': { 'ageGroup': 'D. 35-44', 'canTutoie': True, 'coachingEmailFrequency': 'EMAIL_MAXIMUM', 'frustrations': [], 'gender': 'MASCULINE', 'hasHandicap': False, 'highestDegree': '6 - DEA_DESS_MASTER_PHD', 'origin': 'FROM_A_FRIEND', }, 'project': { 'advices': ['network'], 'readAdvices': ['read-more', 'life-balance'], 'numAdvicesRead': 2, 'job_search_length_months': 4, 'isComplete': True, 'kind': 'FIND_ANOTHER_JOB', 'areaType': 'REGION', 'city': { 'regionName': 'Auvergne-Rhône-Alpes', 'urbanScore': 7, 'urbanContext': '2 - PERIURBAN', }, 'targetJob': { 'domain': 'Commerce, vente et grande distribution', 'name': 'Boulanger', 'job_group': { 'name': 'Boulangerie', }, }, 'feedbackScore': 5, 'feedbackLoveScore': 1, 'minSalary': 45000, 'diagnostic': { 'categoryId': 'stuck-market', } }, 'registeredAt': '2017-07-15T18:06:08Z', 'employmentStatus': { 'bobHasHelped': 'YES', 'bobHasHelpedScore': 1, 'createdAt': '2017-07-25T18:06:08Z', 'daysSinceRegistration': 10, 'otherCoachesUsed': ['PE_COUNSELOR_MEETING', 'MUTUAL_AID_ORGANIZATION'], 'bobRelativePersonalization': 12, }, 'emailsSent': { 'focus-network': 'EMAIL_SENT_CLICKED', 'focus-spontaneous': 'EMAIL_SENT_OPENED', }, 'clientMetrics': { 'firstSessionDurationSeconds': 250, 'isFirstSessionMobile': 'TRUE', }, 'origin': { 'medium': 'ad', 'source': 'facebook', }, 'hasAccount': True, 'isHooked': True, }, json.loads(body))
def test_main(self, mock_randint: mock.MagicMock, mock_simulate: mock.MagicMock) -> None: """Test main.""" mock_randint.return_value = 42 mock_simulate.return_value = [ email_pb2.EmailSent(campaign_id='focus-resume'), ] self.maxDiff = None # pylint: disable=invalid-name self._user_db.user.insert_one({ 'profile': { 'locale': 'fr@tu', 'coachingEmailFrequency': 'EMAIL_MAXIMUM', 'customGender': 'DECLINE_TO_ANSWER', 'email': '*****@*****.**', 'familySituation': 'FAMILY_WITH_KIDS', 'gender': 'MASCULINE', 'isArmyVeteran': False, 'yearOfBirth': 1982, 'highestDegree': 'DEA_DESS_MASTER_PHD', 'origin': 'FROM_A_FRIEND', 'races': ['WHITE'], }, 'projects': [{ 'createdAt': '2018-12-01T00:00:00Z', 'kind': 'FIND_ANOTHER_JOB', 'targetJob': { 'name': 'Boulanger', 'jobGroup': { 'name': 'Boulangerie', 'romeId': 'D0001' }, }, 'areaType': 'REGION', 'city': { 'cityId': '69123', 'name': 'Lyon', 'departementName': 'Rhône', 'regionName': 'Auvergne-Rhône-Alpes', 'urbanScore': 7, }, 'jobSearchStartedAt': '2018-08-01T00:00:00Z', 'minSalary': 45000, 'employmentTypes': ['CDD_OVER_3_MONTHS', 'CDI'], 'trainingFulfillmentEstimate': 'ENOUGH_EXPERIENCE', 'passionateLevel': 'ALIMENTARY_JOB', 'previousJobSimilarity': 'NEVER_DONE', 'seniority': 'NO_SENIORITY', 'networkEstimate': 1, 'weeklyOffersEstimate': 'LESS_THAN_2', 'weeklyApplicationsEstimate': 'SOME', 'totalInterviewsEstimate': 'SOME', 'advices': [ { 'adviceId': 'network', 'numStars': 3, 'status': 'ADVICE_RECOMMENDED', }, { 'adviceId': 'read-more', 'numStars': 1, 'status': 'ADVICE_READ', }, { 'adviceId': 'life-balance', 'numStars': 1, 'numExplorations': 2, 'status': 'ADVICE_READ', }, ], 'actions': [ { 'actionId': 'network', 'status': 'ACTION_UNREAD', }, { 'actionId': 'read-more', 'status': 'ACTION_CURRENT', }, { 'actionId': 'life-balance', 'status': 'ACTION_CURRENT', }, ], 'wasFeedbackRequested': True, 'feedback': { 'score': 5, 'challengeAgreementScore': 1, 'actionPlanHelpsPlanScore': 3, }, 'diagnostic': { 'categoryId': 'stuck-market', }, 'originalSelfDiagnostic': { 'status': 'KNOWN_SELF_DIAGNOSTIC', 'categoryId': 'stuck-market', }, 'strategies': [ { 'strategyId': 'likeable-job' }, { 'strategyId': 'confidence-boost' }, ], 'openedStrategies': [ { 'startedAt': '2018-12-01T00:00:00Z', 'reachedGoals': { 'this goal': True, 'that goal': False }, 'strategyId': 'likeable-job', }, ], }], 'employmentStatus': [{ 'bobHasHelped': 'YES', 'bobRelativePersonalization': 12, 'createdAt': '2017-07-25T18:06:08Z', 'hasBeenPromoted': 'FALSE', 'hasGreaterRole': 'TRUE', 'hasSalaryIncreased': 'TRUE', 'isJobInDifferentSector': 'FALSE', 'otherCoachesUsed': ['PE_COUNSELOR_MEETING', 'MUTUAL_AID_ORGANIZATION'], 'improveSelfConfidenceScore': 3, 'feedback': 'Text feedback that gets scrubbed', }], 'clientMetrics': { 'amplitudeId': '1234ab34f13e5', 'firstSessionDurationSeconds': 250, 'isFirstSessionMobile': 'TRUE', }, 'emailsSent': [ # Ignored because there's another one that was read later. { 'campaignId': 'focus-network', 'sentAt': '2017-10-15T18:06:08Z', 'status': 'EMAIL_SENT_SENT', }, { 'campaignId': 'focus-spontaneous', 'sentAt': '2017-10-15T18:06:08Z', 'status': 'EMAIL_SENT_OPENED', }, { 'campaignId': 'focus-network', 'sentAt': '2017-11-15T18:06:08Z', 'status': 'EMAIL_SENT_CLICKED', }, ], 'origin': { 'source': 'facebook', 'medium': 'ad', 'campaign': 'metiers porteurs', }, 'registeredAt': '2017-07-15T18:06:08Z', 'hasAccount': True, }) self._db.cities.insert_one({ '_id': '69123', 'name': 'Lyon', 'urbanContext': 2, }) db_user = self._user_db.user.find_one() assert db_user user_id = str(db_user['_id']) sync_user_elasticsearch.main(self.mock_elasticsearch, [ '--registered-from', '2017-07', '--no-dry-run', '--disable-sentry' ]) # TODO(cyrille): DRY with _compute_user_data. self.mock_elasticsearch.update.assert_called_once() kwargs = self.mock_elasticsearch.update.call_args[1] body = kwargs.pop('body') self.assertEqual( { 'index': 'bobusers', 'doc_type': '_doc', 'id': user_id, }, kwargs) doc = body.pop('doc') self.assertEqual({'doc_as_upsert': True}, body) # Ensure the document is serializable. self.assertTrue(json.dumps(doc)) self.assertEqual( { 'randomGroup': .42, 'finishedOnboardingPercent': 100, 'profile': { 'ageGroup': 'D. 35-44', 'coachingEmailFrequency': 'EMAIL_MAXIMUM', 'customGender': 'DECLINE_TO_ANSWER', 'familySituation': 'FAMILY_WITH_KIDS', 'frustrations': [], 'gender': 'MASCULINE', 'hasHandicap': False, 'highestDegree': '6 - DEA_DESS_MASTER_PHD', 'isArmyVeteran': False, 'locale': 'fr@tu', 'origin': 'FROM_A_FRIEND', 'races': ['WHITE'], }, 'project': { 'advices': ['network'], 'exploredAdvices': ['life-balance'], 'readAdvices': ['read-more', 'life-balance'], 'numAdvicesRead': 2, 'job_search_length_months': 4, 'isComplete': True, 'kind': 'FIND_ANOTHER_JOB', 'areaType': 'REGION', 'city': { 'regionName': 'Auvergne-Rhône-Alpes', 'urbanScore': 7, 'urbanContext': '2 - PERIURBAN', }, 'targetJob': { 'domain': 'Commerce, vente et grande distribution', 'name': 'Boulanger', 'job_group': { 'name': 'Boulangerie', }, }, 'wasFeedbackRequested': True, 'feedbackScore': 5, 'feedbackLoveScore': 1, 'actionPlanHelpsPlanScore': 3, 'challengeAgreementScore': 0, 'minSalary': 45000, 'employmentTypes': ['CDD_OVER_3_MONTHS', 'CDI'], 'trainingFulfillmentEstimate': 'ENOUGH_EXPERIENCE', 'passionateLevel': 'ALIMENTARY_JOB', 'previousJobSimilarity': 'NEVER_DONE', 'seniority': 'NO_SENIORITY', 'networkEstimate': 'LOW', 'weeklyOffersEstimate': 'LESS_THAN_2', 'weeklyApplicationsEstimate': 'SOME', 'totalInterviewsEstimate': 'SOME', 'diagnostic': { 'categoryId': 'stuck-market', }, 'originalSelfDiagnostic': { 'categoryId': 'stuck-market', 'isSameAsSelf': True, 'status': 'KNOWN_SELF_DIAGNOSTIC', }, 'tocScore': 0, 'ratioOpenedStrategies': .5, 'openedStrategies': ['likeable-job'], 'numStrategiesShown': 2, 'hasReachedAStrategyGoal': True, 'actionPlanStage': 2, 'actionPlanStatus': 'ADDING_ACTIONS', }, 'registeredAt': '2017-07-15T18:06:08Z', 'employmentStatus': { 'bobHasHelped': 'YES', 'bobHasHelpedScore': 1, 'createdAt': '2017-07-25T18:06:08Z', 'daysSinceRegistration': 10, 'hasBeenPromoted': 'FALSE', 'hasGreaterRole': 'TRUE', 'hasSalaryIncreased': 'TRUE', 'isJobInDifferentSector': 'FALSE', 'otherCoachesUsed': ['PE_COUNSELOR_MEETING', 'MUTUAL_AID_ORGANIZATION'], 'bobRelativePersonalization': 12, 'improveSelfConfidenceScore': 3, }, 'emailsSent': { 'focus-network': 'EMAIL_SENT_CLICKED', 'focus-spontaneous': 'EMAIL_SENT_OPENED', }, 'coachingEmails': ['focus-network', 'focus-resume', 'focus-spontaneous'], 'coachingEmailsExpected': 3, 'coachingEmailsSent': 2, 'coachingEmailsClicked': 1, 'coachingEmailsClickedRatio': .5, 'coachingEmailsOpened': 2, 'coachingEmailsOpenedRatio': 1, 'clientMetrics': { 'firstSessionDurationSeconds': 250, 'isFirstSessionMobile': 'TRUE', }, 'origin': { 'medium': 'ad', 'source': 'facebook', 'campaign': 'metiers porteurs', }, 'hasAccount': True, 'isHooked': True, }, doc)