def test_main_function(self, mock_log_info: mock.MagicMock) -> None: """Basic usage.""" self.mongo_db.create_collection('missing-in-importers') self.mongo_db.create_collection('in-both') self.mongo_db.create_collection('in-both-with-meta') two_days_ago = (datetime.datetime.now() - datetime.timedelta(days=2)).replace(microsecond=0) self.mongo_db.meta.insert_one({ '_id': 'in-both-with-meta', 'updated_at': two_days_ago, }) self.mongo_db.create_collection('in-both-not-needed') import_status.main([]) mock_log_info.assert_any_call('%s collection%s without importers:', _AnyColorText('1'), ' is') mock_log_info.assert_any_call('%s collection%s not imported yet:', _AnyColorText('1'), ' is') mock_log_info.assert_any_call( 'Status report on imported collections (%d):', 3) mock_log_info.assert_any_call( '\t%s - %s - %s', _AnyColorText('in-both-not-needed'), _AnyColorText('in both not needed'), termcolor.colored('No import needed', 'green')) mock_log_info.assert_any_call( '\t%s - %s - %s', _AnyColorText('in-both'), _AnyColorText('in both'), termcolor.colored('Metainformation missing', 'red')) mock_log_info.assert_any_call( '\t%s - %s - %s', _AnyColorText('in-both-with-meta'), _AnyColorText('in both with meta (JobGroup)'), _AnyColorText(f'last import: {two_days_ago}'))
def test_run_multiple_importers( self, mock_subprocess_run: mock.MagicMock) -> None: """Run the commands to import multiple collections.""" import_status.main([ '--run', 'collection_id', '--run', 'other_collection_id', ]) self.assertEqual(2, mock_subprocess_run.call_count) mock_subprocess_run.assert_any_call([ 'python', f'{_BOB_EMPLOI_DIR}/data_analysis/importer/my-script-name.py', '--custom_importer_flag', 'value for custom flag', '--mongo_collection', 'collection_id' ], stderr=subprocess.PIPE, check=True) mock_subprocess_run.assert_any_call([ 'python', f'{_BOB_EMPLOI_DIR}/data_analysis/importer/other-script-name.py', '--custom_importer_flag', 'other value for custom flag', '--mongo_collection', 'other_collection_id' ], stderr=subprocess.PIPE, check=True)
def test_run_importer(self, mock_subprocess_run: mock.MagicMock) -> None: """Run the command to import a collection.""" import_status.main(['--run', 'collection_id']) mock_subprocess_run.assert_called_once_with([ 'python', f'{_BOB_EMPLOI_DIR}/data_analysis/importer/my-script-name.py', '--custom_importer_flag', 'value for custom flag', '--mongo_collection', 'collection_id' ], stderr=subprocess.PIPE, check=True)
def test_run_importer_with_extra_args( self, mock_subprocess_run: mock.MagicMock) -> None: """Run the command to import a collection with extra args forwarded.""" import_status.main(['--run', 'collection_id', '--no_diff']) mock_subprocess_run.assert_called_once_with([ 'python', '/work/bob_emploi/data_analysis/importer/my-script-name.py', '--custom_importer_flag', 'value for custom flag', '--mongo_collection', 'collection_id', '--no_diff' ], stderr=subprocess.PIPE, check=True)
def test_display_command(self, mock_log_info: mock.MagicMock) -> None: """Display the command to import a missing collection.""" import_status.main([]) mock_log_info.assert_any_call('%s collection%s not imported yet:', _AnyColorText('1'), ' is') mock_log_info.assert_any_call( 'To import "%s" in "%s", run:\n%s', 'Collection name', 'collection_id', 'docker-compose run --rm -e MONGO_URL=mongodb://test_url/test_db' ' data-analysis-prepare \\\n' ' python bob_emploi/data_analysis/importer/my-script-name.py \\\n' ' --mongo_collection "collection_id"\n')
def test_revert_missing_archive(self, mock_log_error: mock.MagicMock) -> None: """Do nothing when reverting a collection without archive.""" self.mongo_db.collection_id.insert_many([{ '_id': i } for i in range(10)]) import_status.main(['--revert', 'collection_id']) mock_log_error.assert_called_once() self.assertIn('collection_id', mock_log_error.call_args[0]) self.assertEqual( list(range(10)), [doc['_id'] for doc in self.mongo_db.collection_id.find({})])
def test_run_importer(self, pymongo_mock, mock_subprocess_run): """Run the command to import a collection.""" client = mongomock.MongoClient('mongodb://test_url/test_db') pymongo_mock.MongoClient.return_value = client import_status.main(['mongodb://test_url/test_db', '--run', 'collection_id']) mock_subprocess_run.assert_called_once_with([ 'python', '/work/bob_emploi/data_analysis/importer/my-script-name.py', '--custom_importer_flag', 'value for custom flag', '--mongo_url', 'mongodb://test_url/test_db', '--mongo_collection', 'collection_id', ])
def test_personal_database(self, mock_log_info: mock.MagicMock) -> None: """Check division between personal/non personal databases.""" self.mongo_db.create_collection('non-personal') self.mongo_db.create_collection('personal') self.mongo_db.create_collection('personal-no-import') import_status.main([]) mock_log_info.assert_any_call('%s collection%s without importers:', _AnyColorText('1'), ' is') # Although non-personal is imported, it should not be as it's a Personal database. mock_log_info.assert_any_call( 'The collection%s with missing importer%s: %s\n', '', ' is', _AnyColorText("{'non-personal'}"))
def test_display_command_for_specific_collection( self, mock_log_info: mock.MagicMock) -> None: """Display the command to import a specific collection.""" self.mongo_db.create_collection('not_displayed') self.mongo_db.create_collection('collection_id') import_status.main(['collection_id']) mock_log_info.assert_called_once_with( 'To import "%s" in "%s", run:\n%s', 'Collection name', 'collection_id', 'docker-compose run --rm -e MONGO_URL=mongodb://test_url/test_db' ' data-analysis-prepare \\\n' ' python bob_emploi/data_analysis/importer/my-script-name.py \\\n' ' --mongo_collection "collection_id"\n')
def test_display_command(self, pymongo_mock, mock_log_info): """Display the command to import a missing collection.""" client = mongomock.MongoClient('mongodb://test_url/test_db') pymongo_mock.MongoClient.return_value = client import_status.main(['mongodb://test_url/test_db']) mock_log_info.assert_any_call( '%s collection%s not imported yet:', _AnyColorText('1'), ' is') mock_log_info.assert_any_call( 'To import "%s" in "%s", run:\n%s', 'Collection name', 'collection_id', 'docker-compose run --rm data-analysis-prepare \\\n' ' python bob_emploi/data_analysis/importer/my-script-name.py \\\n' ' --mongo_url "mongodb://test_url/test_db" \\\n' ' --mongo_collection "collection_id"\n')
def test_run_importer_fails(self, mock_subprocess_run: mock.MagicMock, mock_log_error: mock.MagicMock) -> None: """Run the command to import a collection.""" mock_subprocess_run.side_effect = subprocess.CalledProcessError( 2, ['the command'], stderr=b'the error') import_status.main(['--run', 'collection_id']) mock_subprocess_run.assert_called_once_with([ 'python', f'{_BOB_EMPLOI_DIR}/data_analysis/importer/my-script-name.py', '--mongo_collection', 'collection_id' ], stderr=subprocess.PIPE, check=True) mock_log_error.assert_any_call( 'Could not import "%s":\nCommand run: %s\nError: %s', 'collection_id', 'the command', 'the error')
def test_display_command(self, mock_log_info: mock.MagicMock) -> None: """Display the command to import a missing collection.""" import_status.main([]) mock_log_info.assert_any_call('%s collection%s not imported yet:', _AnyColorText('1'), ' is') info_log = self._find_log_call_matching('To import "Collection name"', mock_log_info) self.assertIn( 'To import "Collection name" in "collection_id", run:\ndocker-compose run --rm -e', info_log) self.assertNotIn('test_url/test_db', info_log) self.assertRegex( info_log, r'.*docker-compose run --rm .*data-analysis-prepare (\s|\\\n)*' r'python bob_emploi/data_analysis/importer/my-script-name\.py (\s|\\\n)*' r'--mongo_collection "collection_id"\n$')
def test_display_command_for_specific_collection( self, mock_log_info: mock.MagicMock) -> None: """Display the command to import a specific collection.""" self.mongo_db.create_collection('not_displayed') self.mongo_db.create_collection('collection_id') import_status.main(['collection_id']) mock_log_info.assert_called_once() info_log = mock_log_info.call_args[0][0] % mock_log_info.call_args[0][ 1:] self.assertIn( 'To import "Collection name" in "collection_id", run:\ndocker-compose run --rm -e', info_log) self.assertNotIn('test_url/test_db', info_log) self.assertRegex( info_log, r'.*docker-compose run --rm .*data-analysis-prepare (\s|\\\n)*' r'python bob_emploi/data_analysis/importer/my-script-name\.py (\s|\\\n)*' r'--mongo_collection "collection_id"\n$')
def test_importer_with_make_target_fails( self, mock_subprocess_run: mock.MagicMock, mock_log_error: mock.MagicMock) -> None: """Run the command to import a collection with a target to be made.""" fake_command = [ 'python', 'my-folder/my-script.py', '--long-argument', 'value', '--other-arg', 'other-value' ] mock_subprocess_run.side_effect = subprocess.CalledProcessError( 2, fake_command, stderr=b'the error') import_status.main(['--make_data', '--run', 'collection_id']) mock_subprocess_run.assert_called_once_with(['make', 'data/my_target'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) mock_log_error.assert_any_call( 'Could not make "%s":\nCommand run: %s\nError: %s', 'data/my_target', 'python my-folder/my-script.py --long-argument value \\\n --other-arg other-value', 'the error')
def test_run_importer_with_make_target( self, mock_subprocess_run: mock.MagicMock, mock_log_info: mock.MagicMock) -> None: """Run the command to import a collection with a target to be made.""" import_status.main(['--make_data', '--run', 'collection_id']) mock_subprocess_run.assert_any_call(['make', 'data/my_target'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) mock_subprocess_run.assert_any_call([ 'python', f'{_BOB_EMPLOI_DIR}/data_analysis/importer/my-script-name.py', '--needed_data', 'data/my_target', '--mongo_collection', 'collection_id' ], stderr=subprocess.PIPE, check=True) mock_log_info.assert_any_call( 'To make the data file(s) needed by %s importer, run:\n%s', 'Collection name', 'make \\\n' ' data/my_target\n')
def test_main_function(self, pymongo_mock, mock_log_info): """Basic usage.""" client = mongomock.MongoClient('mongodb://test_url/test_db') pymongo_mock.MongoClient.return_value = client mongo_db = client.get_database('test_db') mongo_db.create_collection('missing-in-importers') mongo_db.create_collection('in-both') mongo_db.create_collection('in-both-with-meta') two_days_ago = datetime.datetime.now() - datetime.timedelta(days=2) mongo_db.meta.insert_one({ '_id': 'in-both-with-meta', 'updated_at': two_days_ago, }) mongo_db.create_collection('in-both-not-needed') import_status.main(['mongodb://test_url/test_db']) mock_log_info.assert_any_call( '%s collection%s without importers:', _AnyColorText('1'), ' is') mock_log_info.assert_any_call( '%s collection%s not imported yet:', _AnyColorText('1'), ' is') mock_log_info.assert_any_call( 'Status report on imported collections (%d):', 3) mock_log_info.assert_any_call( '\t%s - %s - %s', _AnyColorText('in-both-not-needed'), _AnyColorText('in both not needed'), termcolor.colored('No import needed', 'green')) mock_log_info.assert_any_call( '\t%s - %s - %s', _AnyColorText('in-both'), _AnyColorText('in both'), termcolor.colored('Metainformation missing', 'red')) mock_log_info.assert_any_call( '\t%s - %s - %s', _AnyColorText('in-both-with-meta'), _AnyColorText('in both with meta (JobGroup)'), _AnyColorText('last import: {}'.format(two_days_ago)))
def test_revert_import(self, mock_log_info: mock.MagicMock) -> None: """Reverting a collection for which there is an archive.""" self.mongo_db.collection_id.insert_many([{ '_id': i } for i in range(10)]) self.mongo_db.get_collection( 'collection_id.2019-03-20_5784037a837ed').insert_many([{ '_id': i } for i in range(10, 20)]) self.mongo_db.get_collection( 'collection_id.2019-03-18_45830e7a865fa').insert_many([{ '_id': i } for i in range(20, 30)]) import_status.main(['--revert', 'collection_id']) mock_log_info.assert_called_with( 'Reverting collection "%s" to version from %s…', 'collection_id', '2019-03-20') self.assertEqual( list(range(10, 20)), [doc['_id'] for doc in self.mongo_db.collection_id.find({})]) self.assertNotIn('collection_id.2019-03-20_5784037a837ed', self.mongo_db.list_collection_names())
def test_run_multiple_importers(self, pymongo_mock, mock_subprocess_run): """Run the commands to import multiple collections.""" client = mongomock.MongoClient('mongodb://test_url/test_db') pymongo_mock.MongoClient.return_value = client import_status.main([ 'mongodb://test_url/test_db', '--run', 'collection_id', '--run', 'other_collection_id', ]) self.assertEqual(2, mock_subprocess_run.call_count) mock_subprocess_run.assert_any_call([ 'python', '/work/bob_emploi/data_analysis/importer/my-script-name.py', '--custom_importer_flag', 'value for custom flag', '--mongo_url', 'mongodb://test_url/test_db', '--mongo_collection', 'collection_id', ]) mock_subprocess_run.assert_any_call([ 'python', '/work/bob_emploi/data_analysis/importer/other-script-name.py', '--custom_importer_flag', 'other value for custom flag', '--mongo_url', 'mongodb://test_url/test_db', '--mongo_collection', 'other_collection_id', ])
def test_details_importer_missing(self) -> None: """Test missing importer.""" # This is the SystemExit from argparse getting a bad argument. with self.assertRaises(SystemExit): import_status.main(['unknown_collection'])
def test_main_unknown_extra_args(self) -> None: """Unknown arg.""" with self.assertRaises(argparse.ArgumentError): import_status.main(['--no_diff'])
def test_main_unknown_extra_args(self) -> None: """Unknown arg.""" with self.assertRaises(SystemExit): import_status.main(['--no_diff'])
def test_details_importer_missing(self) -> None: """Test missing importer.""" with self.assertRaises(argparse.ArgumentError): import_status.main(['unknown_collection'])