def test_should_call_cli_deploy_image_command__with_defaults_and_with_implicit_deployment_config_file(
self, deploy_docker_image_mock):
# given
dc_file = self._touch_file(
'deployment_config.py', '''
from bigflow import Config
deployment_config = Config(name='dev',
properties={
'docker_repository': 'my-docker--repository'
})
''')
# when
cli(['deploy-image', '-v', '0.0.2'])
#then
deploy_docker_image_mock.assert_called_with(
auth_method='local_account',
docker_repository='my-docker--repository',
build_ver='0.0.2',
vault_endpoint=None,
vault_secret=None)
dc_file.unlink()
def test_should_call_cli_deploy_dags_command__with_defaults_and_with_implicit_deployment_config_file(
self, deploy_dags_folder_mock):
#given
dc_file = self._touch_file(
'deployment_config.py', '''
from bigflow import Config
deployment_config = Config(name='dev',
properties={
'gcp_project_id': 'my-gcp-project-id',
'dags_bucket': 'my-dags-bucket',
'vault_secret': 'secret'
})
''')
#when
cli(['deploy-dags'])
#then
deploy_dags_folder_mock.assert_called_with(
auth_method='local_account',
clear_dags_folder=False,
dags_bucket='my-dags-bucket',
dags_dir=self._expected_default_dags_dir(),
project_id='my-gcp-project-id',
vault_endpoint=None,
vault_secret='secret')
dc_file.unlink()
def test_should_call_build_dags_command_through_CLI(self, validate_project_setup_mock, run_process_mock):
# when
cli(['build-dags'])
# then
run_process_mock.assert_any_call([
"python", EXAMPLE_PROJECT_SETUPPY, "build_project", "--build-dags"])
def test_should_call_both_deploy_methods_with_deploy_command(self, deploy_docker_image_mock,
deploy_dags_folder_mock):
# given
self._touch_file('deployment_config.py',
'''
from bigflow import Config
deployment_config = Config(name='dev',
properties={
'docker_repository': 'my-docker--repository',
'gcp_project_id': 'my-gcp-project-id',
'dags_bucket': 'my-dags-bucket'
})
''')
# when
cli(['deploy', '-i', 'my-images/image-version'])
# then
deploy_dags_folder_mock.assert_called_with(auth_method='local_account',
clear_dags_folder=False,
dags_bucket='my-dags-bucket',
dags_dir=self._expected_default_dags_dir(),
project_id='my-gcp-project-id',
vault_endpoint=None,
vault_secret=None)
deploy_docker_image_mock.assert_called_with(auth_method='local_account',
docker_repository='my-docker--repository',
image_tar_path='my-images/image-version',
vault_endpoint=None,
vault_secret=None)
def test_should_call_cli_deploy_dags_command__when_parameters_are_given_by_explicit_deployment_config_file(
self, deploy_dags_folder_mock):
# given
dc_file = self._touch_file(
'deployment_config_another.py', '''
from bigflow import Config
deployment_config = Config(name='dev',
properties={
'gcp_project_id': 'my-another-gcp-project-id',
'vault_endpoint': 'my-another-vault-endpoint',
'dags_bucket': 'my-another-dags-bucket',
'vault_secret': 'secrett'
})
''')
#when
cli([
'deploy-dags', '--deployment-config-path',
dc_file.as_posix(), '--dags-dir', '/tmp/my-dags-dir',
'--auth-method', 'service_account'
])
#then
deploy_dags_folder_mock.assert_called_with(
auth_method='service_account',
clear_dags_folder=False,
dags_bucket='my-another-dags-bucket',
dags_dir='/tmp/my-dags-dir',
project_id='my-another-gcp-project-id',
vault_endpoint='my-another-vault-endpoint',
vault_secret='secrett')
dc_file.unlink()
def test_should_call_cli_start_project_command(self,
cli_start_project_mock):
# when
cli(['start-project'])
# then
self.assertEqual(cli_start_project_mock.call_count, 1)
def test_should_call_cli_build_package_command(self,
_cli_build_package_mock):
# when
cli(['build-package'])
# then
_cli_build_package_mock.assert_called_with()
def test_should_call_cli_build_image_command(self, _cli_build_image_mock):
# when
cli(['build-image'])
# then
_cli_build_image_mock.assert_called_with(
Namespace(operation='build-image'))
def test_should_call_cli_project_version_command_by_alias(
self, get_version):
# when
cli(['pv'])
# then
get_version.assert_called_once()
def test_should_call_cli_deploy_image_command__with_explicit_deployment_config_file(
self, deploy_docker_image_mock):
# given
dc_file = self._touch_file(
'my_deployment_config.py', '''
from bigflow import Config
deployment_config = Config(name='dev',
properties={
'docker_repository': 'my-another-docker-repository' ,
'vault_endpoint' : 'my-another-vault-endpoint'
})
''')
# when
cli([
'deploy-image', '-v', '0.0.3', '--deployment-config-path',
dc_file.as_posix(), '--auth-method', 'service_account',
'--vault-secret', 'secrett'
])
# then
deploy_docker_image_mock.assert_called_with(
auth_method='service_account',
docker_repository='my-another-docker-repository',
build_ver='0.0.3',
vault_endpoint='my-another-vault-endpoint',
vault_secret='secrett')
dc_file.unlink()
def test_should_call_cli_release_command_with_identity_file_parameter_shortcut(
self, release):
# when
cli(['release', '-i', 'path/to/identity_file'])
# then
release.assert_called_once_with('path/to/identity_file')
def test_should_enble_debug_logging_when_verbose_flag_is_specified(self, cli_build):
# when
cli(["--verbose", "build"])
# then
self.assertEqual(len(logging.root.handlers), 1)
self.assertIsInstance(logging.root.handlers[0], logging.StreamHandler)
self.assertEqual(logging.root.level, logging.DEBUG)
def test_should_enble_info_logging(self, cli_build):
# when
cli(["build"])
# then
self.assertEqual(len(logging.root.handlers), 1)
self.assertIsInstance(logging.root.handlers[0], logging.StreamHandler)
self.assertEqual(logging.root.level, logging.INFO)
def test_should_call_build_dags_command_through_CLI(
self, validate_project_setup_mock, run_process_mock):
# when
cli(['build-dags'])
# then
self.assertEqual(run_process_mock.call_count, 1)
run_process_mock.assert_any_call(
'python project_setup.py build_project --build-dags'.split(' '))
def test_should_call_build_image_command_through_CLI(
self, validate_project_setup_mock, find_file_mock,
run_process_mock):
# when
cli(['build-image'])
# then
self.assertEqual(run_process_mock.call_count, 1)
run_process_mock.assert_any_call(
'python project_setup.py build_project --build-image')
def test_should_call_build_command_through_CLI(
self, validate_project_setup_mock, run_process_mock):
# given
validate_project_setup_mock.return_value = '.'
# when
cli(['build'])
# then
run_process_mock.assert_any_call([
"python", EXAMPLE_PROJECT_SETUPPY, "build_project"])
def test_should_call_build_image_command_through_CLI(
self,
read_project_mock: mock.Mock,
build_image_mock: mock.Mock,
):
# when
cli(['build-image'])
# then
read_project_mock.assert_called_once()
build_image_mock.assert_any_call(read_project_mock.return_value)
def test_should_call_cli_build_commands_with_NOW_and_ALL(
self, validate_project_setup_mock, run_process_mock):
# given
validate_project_setup_mock.return_value = '.'
# when
cli(['build','--workflow', 'ALL', '--start-time', 'NOW'])
# then
run_process_mock.assert_any_call([
"python", EXAMPLE_PROJECT_SETUPPY, "build_project"])
def test_should_call_build_command_through_CLI(self,
validate_project_setup_mock,
run_process_mock):
# given
validate_project_setup_mock.return_value = '.'
# when
cli(['build'])
# then
self.assertEqual(run_process_mock.call_count, 1)
run_process_mock.assert_any_call(
'python project_setup.py build_project'.split(' '))
def test_should_call_cli_build_image_command(self, _cli_build_image_mock):
# when
cli(['build-image'])
# then
_cli_build_image_mock.assert_called_with(
Namespace(operation='build-image', export_image_to_file=False))
# when
cli(['build-image', '-e'])
# then
_cli_build_image_mock.assert_called_with(
Namespace(operation='build-image', export_image_to_file=True))
def test_should_call_cli_build_commands_with_NOW_and_ALL(
self,
read_project_spec: mock.Mock,
build_project_mock: mock.Mock,
):
# when
cli(['build','--workflow', 'ALL', '--start-time', 'NOW'])
# then
read_project_spec.assert_called_once()
build_project_mock.assert_any_call(
read_project_spec.return_value,
start_time="2001-02-03 15:00:00",
workflow_id=None,
)
def test_should_call_build_dags_command_through_CLI(
self,
read_project_mock: mock.Mock,
build_dags_mock: mock.Mock,
):
# when
cli(['build-dags'])
# then
read_project_mock.assert_called_once()
build_dags_mock.assert_any_call(
read_project_mock.return_value,
start_time="2001-02-03 15:00:00",
workflow_id=None,
)
def test_should_call_cli_deploy_image_command__when_all_parameters_are_given_by_cli_arguments_and_image_is_loaded_from_tar(
self, deploy_docker_image_mock, load_image_from_tar_mock):
# when
cli([
'deploy-image', '--image-tar-path', 'image-0.0.1.tar',
'--docker-repository', 'my-docker-repository', '--vault-endpoint',
'my-vault-endpoint', '--auth-method', 'service_account',
'--vault-secret', 'secrett'
])
# then
deploy_docker_image_mock.assert_called_with(
auth_method='service_account',
docker_repository='my-docker-repository',
image_tar_path='image-0.0.1.tar',
vault_endpoint='my-vault-endpoint',
vault_secret='secrett')
def test_should_validate_project_setup_before_build(
self, validate_project_setup_mock, run_process_mock):
# when
cli(['build'])
cli(['build-image'])
cli(['build-dags'])
cli(['build-package'])
# then
self.assertEqual(validate_project_setup_mock.call_count, 4)
def test_should_find_tar_in_image_directory(self, deploy_docker_image_mock):
# given
self._touch_file('image-123.tar', '', '.image')
# when
cli(['deploy-image',
'--docker-repository', 'my-docker-repository',
'--vault-endpoint', 'my-vault-endpoint',
'--auth-method', 'vault',
'--vault-secret', 'secrett'
])
# then
deploy_docker_image_mock.assert_called_with(auth_method='vault',
docker_repository='my-docker-repository',
image_tar_path='.image/image-123.tar',
vault_endpoint='my-vault-endpoint',
vault_secret='secrett')
def test_should_call_cli_deploy_dags_command__when_all_parameters_are_given_by_cli_arguments(
self, deploy_dags_folder_mock):
# when
cli([
'deploy-dags', '--dags-bucket', 'my-dags-bucket', '--dags-dir',
'/tmp/my-dags-dir', '--vault-endpoint', 'my-vault-endpoint',
'--gcp-project-id', 'my-gcp-project-id', '--auth-method', 'vault',
'--clear-dags-folder', '--vault-secret', 'secrett'
])
# then
deploy_dags_folder_mock.assert_called_with(
auth_method='vault',
clear_dags_folder=True,
dags_bucket='my-dags-bucket',
dags_dir='/tmp/my-dags-dir',
project_id='my-gcp-project-id',
vault_endpoint='my-vault-endpoint',
vault_secret='secrett')
def test_should_call_cli_build_command(self, _cli_build_mock):
# when
cli(['build'])
# then
_cli_build_mock.assert_called_with(
Namespace(operation='build', start_time=None, workflow=None))
# when
cli(['build', '--start-time', '2020-01-01 00:00:00'])
# then
_cli_build_mock.assert_called_with(
Namespace(operation='build',
start_time='2020-01-01 00:00:00',
workflow=None))
# when
cli([
'build', '--start-time', '2020-01-01 00:00:00', '--workflow',
'some_workflow'
])
# then
_cli_build_mock.assert_called_with(
Namespace(operation='build',
start_time='2020-01-01 00:00:00',
workflow='some_workflow'))
def test_should_call_cli_deploy_dags_command_for_different_environments(
self, deploy_dags_folder_mock):
# given
self._touch_file(
'deployment_config.py', '''
from bigflow import Config
deployment_config = Config(name='dev',
properties={
'gcp_project_id': 'my-gcp-dev-project-id',
'dags_bucket': 'my-dags-dev-bucket',
'vault_secret': 'secret-dev'
})\
.add_configuration(name='prod',
properties={
'gcp_project_id': 'my-gcp-prod-project-id',
'dags_bucket': 'my-dags-prod-bucket',
'vault_secret': 'secret-prod'
})
''')
# when
cli(['deploy-dags'])
# then
deploy_dags_folder_mock.assert_called_with(
auth_method='local_account',
clear_dags_folder=False,
dags_bucket='my-dags-dev-bucket',
dags_dir=self._expected_default_dags_dir(),
project_id='my-gcp-dev-project-id',
vault_endpoint=None,
vault_secret='secret-dev')
# when
cli(['deploy-dags', '--config', 'dev'])
# then
deploy_dags_folder_mock.assert_called_with(
auth_method='local_account',
clear_dags_folder=False,
dags_bucket='my-dags-dev-bucket',
dags_dir=self._expected_default_dags_dir(),
project_id='my-gcp-dev-project-id',
vault_endpoint=None,
vault_secret='secret-dev')
# when
cli(['deploy-dags', '--config', 'prod'])
# then
deploy_dags_folder_mock.assert_called_with(
auth_method='local_account',
clear_dags_folder=False,
dags_bucket='my-dags-prod-bucket',
dags_dir=self._expected_default_dags_dir(),
project_id='my-gcp-prod-project-id',
vault_endpoint=None,
vault_secret='secret-prod')
def test_should_call_cli_deploy_image_command__when_all_parameters_are_given_by_cli_arguments_and_image_is_loaded_from_tar(
self, tag_image, deploy_docker_image_mock,
load_image_from_tar_mock):
#given
tar = self._touch_file('image-0.0.1.tar')
#when
cli([
'deploy-image', '--image-tar-path',
tar.as_posix(), '--docker-repository', 'my-docker-repository',
'--vault-endpoint', 'my-vault-endpoint', '--auth-method',
'service_account', '--vault-secret', 'secrett'
])
#then
deploy_docker_image_mock.assert_called_with(
auth_method='service_account',
docker_repository='my-docker-repository',
build_ver='0.0.1',
vault_endpoint='my-vault-endpoint',
vault_secret='secrett')
tar.unlink()
def test_should_call_cli_build_dags_command(self, _cli_build_dags_mock):
# when
cli(['build-dags'])
# then
_cli_build_dags_mock.assert_called_with(
Namespace(operation='build-dags', start_time=None, workflow=None))
# when
cli(['build-dags', '-t', '2020-01-01 00:00:00'])
# then
_cli_build_dags_mock.assert_called_with(
Namespace(operation='build-dags',
start_time='2020-01-01 00:00:00',
workflow=None))
# when
cli(['build-dags', '-w', 'some_workflow'])
# then
_cli_build_dags_mock.assert_called_with(
Namespace(operation='build-dags',
start_time=None,
workflow='some_workflow'))
# when
cli(['build-dags', '-w', 'some_workflow', '-t', '2020-01-01 00:00:00'])
# then
_cli_build_dags_mock.assert_called_with(
Namespace(operation='build-dags',
start_time='2020-01-01 00:00:00',
workflow='some_workflow'))
# when
cli(['build-dags', '-w', 'some_workflow', '-t', '2020-01-01'])
# then
_cli_build_dags_mock.assert_called_with(
Namespace(operation='build-dags',
start_time='2020-01-01',
workflow='some_workflow'))
# when
with self.assertRaises(SystemExit):
cli(['build-dags', '-w', 'some_workflow', '-t', '20200101'])
