def test_main_function(self, mock_log_info: mock.MagicMock) -> None:
"""Basic usage."""
self.mongo_db.create_collection('missing-in-importers')
self.mongo_db.create_collection('in-both')
self.mongo_db.create_collection('in-both-with-meta')
two_days_ago = (datetime.datetime.now() -
datetime.timedelta(days=2)).replace(microsecond=0)
self.mongo_db.meta.insert_one({
'_id': 'in-both-with-meta',
'updated_at': two_days_ago,
})
self.mongo_db.create_collection('in-both-not-needed')
import_status.main([])
mock_log_info.assert_any_call('%s collection%s without importers:',
_AnyColorText('1'), ' is')
mock_log_info.assert_any_call('%s collection%s not imported yet:',
_AnyColorText('1'), ' is')
mock_log_info.assert_any_call(
'Status report on imported collections (%d):', 3)
mock_log_info.assert_any_call(
'\t%s - %s - %s', _AnyColorText('in-both-not-needed'),
_AnyColorText('in both not needed'),
termcolor.colored('No import needed', 'green'))
mock_log_info.assert_any_call(
'\t%s - %s - %s', _AnyColorText('in-both'),
_AnyColorText('in both'),
termcolor.colored('Metainformation missing', 'red'))
mock_log_info.assert_any_call(
'\t%s - %s - %s', _AnyColorText('in-both-with-meta'),
_AnyColorText('in both with meta (JobGroup)'),
_AnyColorText(f'last import: {two_days_ago}'))
def test_run_multiple_importers(
self, mock_subprocess_run: mock.MagicMock) -> None:
"""Run the commands to import multiple collections."""
import_status.main([
'--run',
'collection_id',
'--run',
'other_collection_id',
])
self.assertEqual(2, mock_subprocess_run.call_count)
mock_subprocess_run.assert_any_call([
'python',
f'{_BOB_EMPLOI_DIR}/data_analysis/importer/my-script-name.py',
'--custom_importer_flag', 'value for custom flag',
'--mongo_collection', 'collection_id'
],
stderr=subprocess.PIPE,
check=True)
mock_subprocess_run.assert_any_call([
'python',
f'{_BOB_EMPLOI_DIR}/data_analysis/importer/other-script-name.py',
'--custom_importer_flag', 'other value for custom flag',
'--mongo_collection', 'other_collection_id'
],
stderr=subprocess.PIPE,
check=True)
def test_run_importer(self, mock_subprocess_run: mock.MagicMock) -> None:
"""Run the command to import a collection."""
import_status.main(['--run', 'collection_id'])
mock_subprocess_run.assert_called_once_with([
'python',
f'{_BOB_EMPLOI_DIR}/data_analysis/importer/my-script-name.py',
'--custom_importer_flag', 'value for custom flag',
'--mongo_collection', 'collection_id'
],
stderr=subprocess.PIPE,
check=True)
def test_run_importer_with_extra_args(
self, mock_subprocess_run: mock.MagicMock) -> None:
"""Run the command to import a collection with extra args forwarded."""
import_status.main(['--run', 'collection_id', '--no_diff'])
mock_subprocess_run.assert_called_once_with([
'python',
'/work/bob_emploi/data_analysis/importer/my-script-name.py',
'--custom_importer_flag', 'value for custom flag',
'--mongo_collection', 'collection_id', '--no_diff'
],
stderr=subprocess.PIPE,
check=True)
def test_display_command(self, mock_log_info: mock.MagicMock) -> None:
"""Display the command to import a missing collection."""
import_status.main([])
mock_log_info.assert_any_call('%s collection%s not imported yet:',
_AnyColorText('1'), ' is')
mock_log_info.assert_any_call(
'To import "%s" in "%s", run:\n%s', 'Collection name',
'collection_id',
'docker-compose run --rm -e MONGO_URL=mongodb://test_url/test_db'
' data-analysis-prepare \\\n'
' python bob_emploi/data_analysis/importer/my-script-name.py \\\n'
' --mongo_collection "collection_id"\n')
def test_revert_missing_archive(self,
mock_log_error: mock.MagicMock) -> None:
"""Do nothing when reverting a collection without archive."""
self.mongo_db.collection_id.insert_many([{
'_id': i
} for i in range(10)])
import_status.main(['--revert', 'collection_id'])
mock_log_error.assert_called_once()
self.assertIn('collection_id', mock_log_error.call_args[0])
self.assertEqual(
list(range(10)),
[doc['_id'] for doc in self.mongo_db.collection_id.find({})])
def test_run_importer(self, pymongo_mock, mock_subprocess_run):
"""Run the command to import a collection."""
client = mongomock.MongoClient('mongodb://test_url/test_db')
pymongo_mock.MongoClient.return_value = client
import_status.main(['mongodb://test_url/test_db', '--run', 'collection_id'])
mock_subprocess_run.assert_called_once_with([
'python', '/work/bob_emploi/data_analysis/importer/my-script-name.py',
'--custom_importer_flag', 'value for custom flag',
'--mongo_url', 'mongodb://test_url/test_db',
'--mongo_collection', 'collection_id',
])
def test_personal_database(self, mock_log_info: mock.MagicMock) -> None:
"""Check division between personal/non personal databases."""
self.mongo_db.create_collection('non-personal')
self.mongo_db.create_collection('personal')
self.mongo_db.create_collection('personal-no-import')
import_status.main([])
mock_log_info.assert_any_call('%s collection%s without importers:',
_AnyColorText('1'), ' is')
# Although non-personal is imported, it should not be as it's a Personal database.
mock_log_info.assert_any_call(
'The collection%s with missing importer%s: %s\n', '', ' is',
_AnyColorText("{'non-personal'}"))
def test_display_command_for_specific_collection(
self, mock_log_info: mock.MagicMock) -> None:
"""Display the command to import a specific collection."""
self.mongo_db.create_collection('not_displayed')
self.mongo_db.create_collection('collection_id')
import_status.main(['collection_id'])
mock_log_info.assert_called_once_with(
'To import "%s" in "%s", run:\n%s', 'Collection name',
'collection_id',
'docker-compose run --rm -e MONGO_URL=mongodb://test_url/test_db'
' data-analysis-prepare \\\n'
' python bob_emploi/data_analysis/importer/my-script-name.py \\\n'
' --mongo_collection "collection_id"\n')
def test_display_command(self, pymongo_mock, mock_log_info):
"""Display the command to import a missing collection."""
client = mongomock.MongoClient('mongodb://test_url/test_db')
pymongo_mock.MongoClient.return_value = client
import_status.main(['mongodb://test_url/test_db'])
mock_log_info.assert_any_call(
'%s collection%s not imported yet:', _AnyColorText('1'), ' is')
mock_log_info.assert_any_call(
'To import "%s" in "%s", run:\n%s',
'Collection name', 'collection_id',
'docker-compose run --rm data-analysis-prepare \\\n'
' python bob_emploi/data_analysis/importer/my-script-name.py \\\n'
' --mongo_url "mongodb://test_url/test_db" \\\n'
' --mongo_collection "collection_id"\n')
def test_run_importer_fails(self, mock_subprocess_run: mock.MagicMock,
mock_log_error: mock.MagicMock) -> None:
"""Run the command to import a collection."""
mock_subprocess_run.side_effect = subprocess.CalledProcessError(
2, ['the command'], stderr=b'the error')
import_status.main(['--run', 'collection_id'])
mock_subprocess_run.assert_called_once_with([
'python',
f'{_BOB_EMPLOI_DIR}/data_analysis/importer/my-script-name.py',
'--mongo_collection', 'collection_id'
],
stderr=subprocess.PIPE,
check=True)
mock_log_error.assert_any_call(
'Could not import "%s":\nCommand run: %s\nError: %s',
'collection_id', 'the command', 'the error')
def test_display_command(self, mock_log_info: mock.MagicMock) -> None:
"""Display the command to import a missing collection."""
import_status.main([])
mock_log_info.assert_any_call('%s collection%s not imported yet:',
_AnyColorText('1'), ' is')
info_log = self._find_log_call_matching('To import "Collection name"',
mock_log_info)
self.assertIn(
'To import "Collection name" in "collection_id", run:\ndocker-compose run --rm -e',
info_log)
self.assertNotIn('test_url/test_db', info_log)
self.assertRegex(
info_log,
r'.*docker-compose run --rm .*data-analysis-prepare (\s|\\\n)*'
r'python bob_emploi/data_analysis/importer/my-script-name\.py (\s|\\\n)*'
r'--mongo_collection "collection_id"\n$')
def test_display_command_for_specific_collection(
self, mock_log_info: mock.MagicMock) -> None:
"""Display the command to import a specific collection."""
self.mongo_db.create_collection('not_displayed')
self.mongo_db.create_collection('collection_id')
import_status.main(['collection_id'])
mock_log_info.assert_called_once()
info_log = mock_log_info.call_args[0][0] % mock_log_info.call_args[0][
1:]
self.assertIn(
'To import "Collection name" in "collection_id", run:\ndocker-compose run --rm -e',
info_log)
self.assertNotIn('test_url/test_db', info_log)
self.assertRegex(
info_log,
r'.*docker-compose run --rm .*data-analysis-prepare (\s|\\\n)*'
r'python bob_emploi/data_analysis/importer/my-script-name\.py (\s|\\\n)*'
r'--mongo_collection "collection_id"\n$')
def test_importer_with_make_target_fails(
self, mock_subprocess_run: mock.MagicMock,
mock_log_error: mock.MagicMock) -> None:
"""Run the command to import a collection with a target to be made."""
fake_command = [
'python', 'my-folder/my-script.py', '--long-argument', 'value',
'--other-arg', 'other-value'
]
mock_subprocess_run.side_effect = subprocess.CalledProcessError(
2, fake_command, stderr=b'the error')
import_status.main(['--make_data', '--run', 'collection_id'])
mock_subprocess_run.assert_called_once_with(['make', 'data/my_target'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=True)
mock_log_error.assert_any_call(
'Could not make "%s":\nCommand run: %s\nError: %s',
'data/my_target',
'python my-folder/my-script.py --long-argument value \\\n --other-arg other-value',
'the error')
def test_run_importer_with_make_target(
self, mock_subprocess_run: mock.MagicMock,
mock_log_info: mock.MagicMock) -> None:
"""Run the command to import a collection with a target to be made."""
import_status.main(['--make_data', '--run', 'collection_id'])
mock_subprocess_run.assert_any_call(['make', 'data/my_target'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=True)
mock_subprocess_run.assert_any_call([
'python',
f'{_BOB_EMPLOI_DIR}/data_analysis/importer/my-script-name.py',
'--needed_data', 'data/my_target', '--mongo_collection',
'collection_id'
],
stderr=subprocess.PIPE,
check=True)
mock_log_info.assert_any_call(
'To make the data file(s) needed by %s importer, run:\n%s',
'Collection name', 'make \\\n'
' data/my_target\n')
def test_main_function(self, pymongo_mock, mock_log_info):
"""Basic usage."""
client = mongomock.MongoClient('mongodb://test_url/test_db')
pymongo_mock.MongoClient.return_value = client
mongo_db = client.get_database('test_db')
mongo_db.create_collection('missing-in-importers')
mongo_db.create_collection('in-both')
mongo_db.create_collection('in-both-with-meta')
two_days_ago = datetime.datetime.now() - datetime.timedelta(days=2)
mongo_db.meta.insert_one({
'_id': 'in-both-with-meta',
'updated_at': two_days_ago,
})
mongo_db.create_collection('in-both-not-needed')
import_status.main(['mongodb://test_url/test_db'])
mock_log_info.assert_any_call(
'%s collection%s without importers:', _AnyColorText('1'), ' is')
mock_log_info.assert_any_call(
'%s collection%s not imported yet:', _AnyColorText('1'), ' is')
mock_log_info.assert_any_call(
'Status report on imported collections (%d):', 3)
mock_log_info.assert_any_call(
'\t%s - %s - %s',
_AnyColorText('in-both-not-needed'),
_AnyColorText('in both not needed'),
termcolor.colored('No import needed', 'green'))
mock_log_info.assert_any_call(
'\t%s - %s - %s',
_AnyColorText('in-both'),
_AnyColorText('in both'),
termcolor.colored('Metainformation missing', 'red'))
mock_log_info.assert_any_call(
'\t%s - %s - %s',
_AnyColorText('in-both-with-meta'),
_AnyColorText('in both with meta (JobGroup)'),
_AnyColorText('last import: {}'.format(two_days_ago)))
def test_revert_import(self, mock_log_info: mock.MagicMock) -> None:
"""Reverting a collection for which there is an archive."""
self.mongo_db.collection_id.insert_many([{
'_id': i
} for i in range(10)])
self.mongo_db.get_collection(
'collection_id.2019-03-20_5784037a837ed').insert_many([{
'_id': i
} for i in range(10, 20)])
self.mongo_db.get_collection(
'collection_id.2019-03-18_45830e7a865fa').insert_many([{
'_id': i
} for i in range(20, 30)])
import_status.main(['--revert', 'collection_id'])
mock_log_info.assert_called_with(
'Reverting collection "%s" to version from %s…', 'collection_id',
'2019-03-20')
self.assertEqual(
list(range(10, 20)),
[doc['_id'] for doc in self.mongo_db.collection_id.find({})])
self.assertNotIn('collection_id.2019-03-20_5784037a837ed',
self.mongo_db.list_collection_names())
def test_run_multiple_importers(self, pymongo_mock, mock_subprocess_run):
"""Run the commands to import multiple collections."""
client = mongomock.MongoClient('mongodb://test_url/test_db')
pymongo_mock.MongoClient.return_value = client
import_status.main([
'mongodb://test_url/test_db',
'--run', 'collection_id',
'--run', 'other_collection_id',
])
self.assertEqual(2, mock_subprocess_run.call_count)
mock_subprocess_run.assert_any_call([
'python', '/work/bob_emploi/data_analysis/importer/my-script-name.py',
'--custom_importer_flag', 'value for custom flag',
'--mongo_url', 'mongodb://test_url/test_db',
'--mongo_collection', 'collection_id',
])
mock_subprocess_run.assert_any_call([
'python', '/work/bob_emploi/data_analysis/importer/other-script-name.py',
'--custom_importer_flag', 'other value for custom flag',
'--mongo_url', 'mongodb://test_url/test_db',
'--mongo_collection', 'other_collection_id',
])
def test_details_importer_missing(self) -> None:
"""Test missing importer."""
# This is the SystemExit from argparse getting a bad argument.
with self.assertRaises(SystemExit):
import_status.main(['unknown_collection'])
def test_main_unknown_extra_args(self) -> None:
"""Unknown arg."""
with self.assertRaises(argparse.ArgumentError):
import_status.main(['--no_diff'])
def test_main_unknown_extra_args(self) -> None:
"""Unknown arg."""
with self.assertRaises(SystemExit):
import_status.main(['--no_diff'])
def test_details_importer_missing(self) -> None:
"""Test missing importer."""
with self.assertRaises(argparse.ArgumentError):
import_status.main(['unknown_collection'])
