def test_pull_retry(self):
workflow = self.workflow()
tasker = MockerTasker()
expected_pullspec = '%s/%s' % (self.CRANE_URI, self.TEST_UNIQUE_IMAGE)
test_id = 'sha256:(new)'
(flexmock(tasker).should_call('pull_image').and_return(
expected_pullspec).times(3))
(flexmock(tasker).should_receive('inspect_image').with_args(
expected_pullspec).and_raise(
NotFound('message', flexmock(content=None))).and_raise(
NotFound('message', flexmock(content=None))).and_return({
'Id':
test_id
}).times(3))
plugin = PulpPullPlugin(tasker, workflow, timeout=1, retry_delay=0.6)
# Plugin return value is the new ID
assert plugin.run() == test_id
assert len(tasker.pulled_images) == 3
for image in tasker.pulled_images:
pulled = image.to_str()
assert pulled == expected_pullspec
# Image ID is updated in workflow
assert workflow.builder.image_id == test_id
def test_down_with_no_resources(self):
project = Project.from_config(
name='test',
client=self.mock_client,
config_data=Config(
project_name='',
version=V2_0,
services=[{
'name': 'web',
'image': BUSYBOX_IMAGE_WITH_TAG,
}],
networks={'default': {}},
volumes={'data': {}},
secrets=None,
configs=None,
),
)
self.mock_client.remove_network.side_effect = NotFound(
None, None, 'oops')
self.mock_client.remove_volume.side_effect = NotFound(
None, None, 'oops')
project.down(ImageType.all, True)
self.mock_client.remove_image.assert_called_once_with(
BUSYBOX_IMAGE_WITH_TAG)
def test_ensure_socat_running_some_wait(mocker, socat_mocked):
mocker.patch.object(socat_mocked.client.containers, 'get').side_effect = [
NotFound('not found'),
NotFound('not found'),
mocker.MagicMock(status='creating'),
mocker.MagicMock(status='creating'),
mocker.MagicMock(status='creating'),
mocker.MagicMock(status='running')
]
# noinspection PyProtectedMember
socat_mocked._ensure_socat_running()
# noinspection PyUnresolvedReferences
assert socat_mocked.sleep.call_count == 5
def unregister_container(instance: object, name: str) -> None:
if name not in instance._containers:
raise NotFound(
f"{instance}: there is no container registered for name `{name}'"
)
del instance._containers[name]
LOGGER.debug("%s: container `%s' unregistered", instance, name)
def run_couchdb_docker_container(docker_client):
# if there is no image we pull it
try:
couchdb_image = docker_client.images.get("couchdb")
except NotFound as e:
couchdb_image = docker_client.images.pull("couchdb", tag="latest")
assert couchdb_image is not None
# if container is not running we will start it
try:
couch_container = docker_client.containers.get("appguard_couch")
if couch_container.status == "exited" or (couch_container.status
== "created"):
couch_container.remove()
raise NotFound("Container Exited or could not be started")
except NotFound as e:
print("Couch docker container not found or not running\n")
couch_container = docker_client.containers.run(
"couchdb",
name="appguard_couch",
ports={"5984/tcp": 5984},
detach=True)
time.sleep(5)
assert couch_container.status == "running" or (
couch_container.status == "created"
and not couch_container.attrs["State"]["Error"])
return couch_container
def compose_project_by_name(project_name: str) -> ComposeProject:
project_config: ComposeProjectConfig = compose_config()
if project_config.project_name == project_name:
return ComposeProject(project_name=project_name,
project_config=project_config)
else:
message = f"couldn't find project '{project_name}'"
raise NotFound(message="Not Found", explanation=message)
def mock_env(tmpdir, docker_tasker,
has_appregistry_label=False, appregistry_label=False,
has_bundle_label=True, bundle_label=True,
has_archive=True,
scratch=False, orchestrator=False, selected_platform=True, empty_archive=False,
remove_fails=False):
build_json = {'metadata': {'labels': {'scratch': scratch}}}
flexmock(util).should_receive('get_build_json').and_return(build_json)
mock_dockerfile(
tmpdir,
has_appregistry_label=has_appregistry_label, appregistry_label=appregistry_label,
has_bundle_label=has_bundle_label, bundle_label=bundle_label
)
workflow = mock_workflow(tmpdir, for_orchestrator=orchestrator)
mock_stream = generate_archive(tmpdir, empty_archive)
plugin_conf = [{'name': ExportOperatorManifestsPlugin.key}]
if selected_platform:
plugin_conf[0]['args'] = {'operator_manifests_extract_platform': machine(),
'platform': machine()}
runner = PostBuildPluginsRunner(docker_tasker, workflow, plugin_conf)
(flexmock(docker_tasker.tasker.d.wrapped)
.should_receive('create_container')
.with_args(workflow.image, command=["/bin/bash"])
.and_return({'Id': CONTAINER_ID}))
if remove_fails:
(flexmock(docker_tasker.tasker.d.wrapped)
.should_receive('remove_container')
.with_args(CONTAINER_ID)
.and_raise(Exception('error')))
else:
(flexmock(docker_tasker.tasker.d.wrapped)
.should_receive('remove_container')
.with_args(CONTAINER_ID))
if has_archive:
(flexmock(docker_tasker.tasker.d.wrapped)
.should_receive('get_archive')
.with_args(CONTAINER_ID, '/manifests')
.and_return(mock_stream, {}))
elif has_archive is not None:
response = Response()
response.status_code = 404
(flexmock(docker_tasker.tasker.d.wrapped)
.should_receive('get_archive')
.with_args(CONTAINER_ID, '/manifests')
.and_raise(NotFound('Not found', response=response)))
else:
response = Response()
response.status_code = 500
(flexmock(docker_tasker.tasker.d.wrapped)
.should_receive('get_archive')
.with_args(CONTAINER_ID, '/manifests')
.and_raise(Exception('error')))
return runner
def __call__(self, cmd, timeout=10):
deprecation("consider to use Docker Python module instead of using Docker CLI commands")
res = LOCALRUNNER.run('docker {}'.format(cmd), ignore_status=True, timeout=timeout)
if res.exit_status:
if 'No such container:' in res.stderr:
raise NotFound(res.stderr)
raise DockerException('command: {}, error: {}, output: {}'.format(cmd, res.stderr, res.stdout))
return res
def get_container(instance: object, name, raise_not_found_exc: bool = True) -> Optional[Container]:
assert name is not None, "None is not allowed as a container name"
container = instance._containers.get(name)
if container is None and raise_not_found_exc:
raise NotFound(f"There is no container `{name}' for {instance}.")
if container:
container.reload()
return container
def test_down_with_no_resources(self):
project = Project.from_config(
name='test',
client=self.mock_client,
config_data=Config(
version='2',
services=[{
'name': 'web',
'image': 'busybox:latest',
}],
networks={'default': {}},
volumes={'data': {}},
),
)
self.mock_client.remove_network.side_effect = NotFound(None, None, 'oops')
self.mock_client.remove_volume.side_effect = NotFound(None, None, 'oops')
project.down(ImageType.all, True)
self.mock_client.remove_image.assert_called_once_with("busybox:latest")
def compose_service(project_name: str, service_name: str) -> ComposeService:
try:
service_config: ComposeServiceConfig = compose_config(
).get_service_config(project_name=project_name,
service_name=service_name)
return ComposeService(project_name=project_name,
service_name=service_name,
service_config=service_config)
except StopIteration:
message = f"couldn't find service '{service_name}' in project '{project_name}'"
raise NotFound(message="Not Found", explanation=message)
def test_must_work_when_container_is_not_found(self):
self.container.is_created.return_value = True
real_container_mock = Mock()
self.mock_docker_client.containers.get.side_effect = NotFound("msg")
real_container_mock.remove = Mock()
self.container.stop()
self.mock_docker_client.containers.get.assert_called_with("someid")
real_container_mock.remove.assert_not_called()
# Ensure ID remains set
self.assertIsNotNone(self.container.id)
def test_check_and_download_remote_plugin_not_installed(mock_setting_get_instance, mock_get_xtables_lock_mount,
mock_configure_xtables_mount, docker_plugin, mock_plugin,
mock_setting):
mock_setting.remote_url = "http://remote-url.kt"
mock_setting_get_instance.return_value = mock_setting
docker_plugin.client.plugins.get.return_value = None
docker_plugin.client.plugins.get.side_effect = NotFound('Fail')
with pytest.raises(Exception) as e:
docker_plugin.check_and_download_plugin()
assert str(e.value) == "Kathara Network Plugin not found on remote Docker connection."
assert not mock_get_xtables_lock_mount.called
assert not mock_configure_xtables_mount.called
def test_check_and_download_plugin_not_found(mock_setting_get_instance, mock_get_xtables_lock_mount,
docker_plugin, mock_plugin, mock_setting):
mock_setting_get_instance.return_value = mock_setting
docker_plugin.client.plugins.get.return_value = None
docker_plugin.client.plugins.get.side_effect = NotFound('Fail')
docker_plugin.client.plugins.install.return_value = mock_plugin
mock_plugin.enabled = False
docker_plugin.check_and_download_plugin()
docker_plugin.client.plugins.get.assert_called_once_with("kathara/katharanp:" + utils.get_architecture())
assert not mock_plugin.upgrade.called
mock_get_xtables_lock_mount.assert_called_once()
mock_plugin.enable.assert_called_once()
assert not mock_plugin.disable.called
def test_must_work_when_container_is_not_found(self):
self.container.is_created.return_value = True
real_container_mock = Mock()
self.mock_docker_client.containers.get.side_effect = NotFound("msg")
real_container_mock.remove = Mock()
self.container.delete()
self.mock_docker_client.containers.get.assert_called_with("someid")
real_container_mock.remove.assert_not_called()
# Must reset ID to None because container is now gone
self.assertIsNone(self.container.id)
def test_pull_timeout(self):
workflow = self.workflow()
tasker = MockerTasker()
(flexmock(tasker).should_call('pull_image').and_return(
self.EXPECTED_PULLSPEC).times(3))
(flexmock(tasker).should_receive('inspect_image').with_args(
self.EXPECTED_PULLSPEC).and_raise(
NotFound('message', flexmock(content=None))).times(3))
plugin = PulpPullPlugin(tasker, workflow, timeout=1, retry_delay=0.6)
# Should raise a timeout exception
with pytest.raises(CraneTimeoutError):
plugin.run()
def parse_inspect_output(out, item_type):
"""
Parses the output of the Docker CLI 'docker inspect <container>' or 'docker network inspect <network>'. Essentially
just returns the parsed JSON string, like the Docker API does.
:param out: CLI output.
:type out: unicode | str
:param item_type: Type of the item that has been inspected (e.g. 'container').
:type item_type: unicode | str
:return: Parsed result.
:rtype: dict
"""
parsed = json.loads(out, encoding='utf-8')
if parsed:
return parsed[0]
raise NotFound("{0} not found.".format(item_type.title()), None)
def test_pull_timeout(self):
workflow = self.workflow()
tasker = MockerTasker()
expected_pullspec = '%s/%s' % (self.CRANE_URI, self.TEST_UNIQUE_IMAGE)
(flexmock(tasker).should_call('pull_image').and_return(
expected_pullspec).times(3))
(flexmock(tasker).should_receive('inspect_image').with_args(
expected_pullspec).and_raise(
NotFound('message', flexmock(content=None))).times(3))
plugin = PulpPullPlugin(tasker, workflow, timeout=1, retry_delay=0.6)
# Should raise a timeout exception
with pytest.raises(CraneTimeoutError):
plugin.run()
def get_container(cid):
if cid == 'eeeee':
raise NotFound(None, None, "oops")
if cid == 'abcde':
name = 'web'
labels = {LABEL_SERVICE: name}
elif cid == 'ababa':
name = 'db'
labels = {LABEL_SERVICE: name}
else:
labels = {}
name = ''
return {
'Id': cid,
'Config': {'Labels': labels},
'Name': '/project_%s_1' % name,
}
def run_opa_docker_container(docker_client):
"""
We will use this function for system testing
:param docker_client: Docker client handle
:type docker_client:
:return: docker container handle
:rtype: string
"""
# if there is no image we pull it
try:
opa_image = docker_client.images.get("openpolicyagent/opa")
except NotFound as e:
opa_image = docker_client.images.pull("openpolicyagent/opa",
tag="latest")
assert opa_image is not None
# if container is not running we will start it
try:
# TODO: change name to something more appropriate
opa_container = docker_client.containers.get("appguard_opa")
if opa_container.status == "exited" or (opa_container.status
== "created"):
opa_container.remove()
raise NotFound("Container Exited or could not be started")
except NotFound as e:
print("OPA docker container not found or not running. \n")
opa_container = docker_client.containers.run(
"openpolicyagent/opa",
command="run --server --log-level=debug",
name="appguard_opa",
ports={"8181/tcp": 8181},
detach=True)
time.sleep(5)
assert opa_container.status == "running" or (
opa_container.status == "created"
and not opa_container.attrs["State"]["Error"])
return opa_container
class JobModelTest(TestCase):
def setUp(self):
self.project = models.Project.objects.create(
name='abdallah', url='https://github.com/ZuluPro/abdallah.git')
self.build = models.Build.objects.create(project=self.project)
def test_create(self):
job = models.Job.objects.create(build=self.build)
self.assertEqual(job.number, 1)
def test_get_absolute_url(self):
job = models.Job.objects.create(build=self.build)
url = job.get_absolute_url()
self.assertIsInstance(url, six.string_types)
def test_get_css_class(self):
job = models.Job.objects.create(build=self.build)
suffix = job.get_css_class()
self.assertIsInstance(suffix, six.string_types)
def test_container_name(self):
job = models.Job.objects.create(build=self.build)
self.assertIsInstance(job.container_name, six.string_types)
@mock.patch('docker.Client.logs', return_value='FOO-BAR')
def test_logs(self, *args):
job = models.Job.objects.create(build=self.build)
logs = job.logs
self.assertIsInstance(logs, six.string_types)
self.assertEqual(logs, 'FOO-BAR')
@mock.patch('docker.Client.logs',
side_effect=NotFound('foo', mock.MagicMock()))
def test_logs_not_found_container(self, *args):
job = models.Job.objects.create(build=self.build)
logs = job.logs
self.assertIsInstance(logs, six.string_types)
def raises_NotFound(name: str):
raise NotFound("")
def main(args):
#: setup parser -----------------------------------------------------------
parser = argparse.ArgumentParser(description='Magen IO Server',
usage=("\npython3 server.py "
"--csrf"
"--clean-init"
"--ingestion-data-dir"
"\n\nnote:\n"
"root privileges are required "))
if inside_docker():
ingestion_data_dir = os.path.join("/opt", "data")
else:
home_dir = str(Path.home())
ingestion_data_dir = os.path.join(home_dir, "magen_data", "ingestion")
parser.add_argument('--ingestion-data-dir',
default=ingestion_data_dir,
help='Set directory for data files'
'Default is %s' % ingestion_data_dir)
parser.add_argument('--clean-init',
action='store_false',
help='Clean All data when initializing'
'Default is to clean)')
parser.add_argument('--csrf',
action='store_true',
help='Enable Cross Request Forgery protection'
'Default is to not use it)')
parser.add_argument('--test',
action='store_true',
help='Run server in test mode. Used for unit tests'
'Default is to run in production mode)')
#: parse CMD arguments ----------------------------------------------------
# args = parser.parse_args()
args, _ = parser.parse_known_args(args)
""" Main Magen.io Sever """
home_dir = str(Path.home())
ingestion_globals = IngestionGlobals()
ingestion_globals.data_dir = args.ingestion_data_dir
try:
os.makedirs(ingestion_globals.data_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Starting OPA server
docker_client = docker.from_env()
# if there is no image we pull it
try:
opa_image = docker_client.images.get("openpolicyagent/opa")
except NotFound as e:
opa_image = docker_client.images.pull("openpolicyagent/opa",
tag="latest")
assert opa_image is not None
# if container is not running we will start it
try:
opa_container = docker_client.containers.get("magen_opa")
if opa_container.status == "exited" or opa_container.status == "created":
opa_container.remove()
raise NotFound("Container Exited or could not be started")
except NotFound as e:
print("OPA docker container not found or not running, starting... \n")
opa_container = docker_client.containers.run(
"openpolicyagent/opa",
command="run --server --log-level=debug",
name="magen_opa",
ports={"8181/tcp": 8181},
detach=True)
time.sleep(5)
assert opa_container.status == "running" or (
opa_container.status == "created"
and not opa_container.attrs["State"]["Error"])
ks_args = [[]]
ks_process = Process(target=ks_server.main, args=ks_args)
ks_process.start()
time.sleep(2)
serverurls = ServerUrls()
ks_check_url = serverurls.key_server_base_url + "check/"
get_resp_obj = RestClientApis.http_get_and_check_success(ks_check_url)
assert get_resp_obj.success is True
mongo_ip, mongo_port = mongo_host_port()
# We initialize at runtime everything about Mongo and its functions
# Any client of the API can change it later
db = MainDb.get_instance()
db.core_database = MongoCore.get_instance()
db.core_database.utils_strategy = MongoUtils.get_instance()
db.core_database.asset_strategy = MongoAsset.get_instance()
db.core_database.db_ip_port = '{ip}:{port}'.format(ip=mongo_ip,
port=mongo_port)
db.core_database.utils_strategy.check_db(db.core_database.db_ip_port)
db.core_database.initialize()
if args.clean_init:
success, _ = AssetDbApi.delete_all()
assert success is True
user_api.drop_user_collection()
GridFsApi.delete_all()
if args.csrf:
app.config['WTF_CSRF_ENABLED'] = True
app.config['WTF_CSRF_SECRET_KEY'] = token_hex(16)
CSRFProtect(app)
else:
app.config['WTF_CSRF_ENABLED'] = False
app.register_blueprint(main_bp)
app.register_blueprint(users_bp)
app.register_blueprint(ingestion_file_upload_bp,
url_prefix='/magen/ingestion/v2')
app.register_blueprint(ingestion_bp_v2, url_prefix='/magen/ingestion/v2')
app.register_blueprint(ingestion_bp, url_prefix='/magen/ingestion/v2')
app.run('0.0.0.0', 5005, threaded=True)
def _container_not_exists(name):
raise NotFound(f'container: {name} not exists')
def mock_env(tmpdir,
docker_tasker,
has_appregistry_label=False,
appregistry_label=False,
has_bundle_label=True,
bundle_label=True,
has_archive=True,
scratch=False,
orchestrator=False,
selected_platform=True,
empty_archive=False,
remove_fails=False,
change_csv_content=False,
multiple_csv=False):
repo_dir = tmpdir.join('test-operator').mkdir()
mock_dockerfile(repo_dir,
has_appregistry_label=has_appregistry_label,
appregistry_label=appregistry_label,
has_bundle_label=has_bundle_label,
bundle_label=bundle_label)
manifests_dir = mock_manifests_dir(repo_dir)
env = (MockEnv().for_plugin(
'postbuild', ExportOperatorManifestsPlugin.key).set_scratch(scratch))
if orchestrator:
env.make_orchestrator()
class MockSource(object):
@property
def manifests_dir(self):
return manifests_dir
# Set a new source object, only the manifests_dir property is required for tests.
setattr(env.workflow, 'source', MockSource())
env.workflow.builder.set_df_path(str(repo_dir))
mock_stream = generate_archive(tmpdir, empty_archive, change_csv_content,
multiple_csv)
if selected_platform:
env.set_plugin_args({
'operator_manifests_extract_platform': machine(),
'platform': machine()
})
(flexmock(docker_tasker.tasker.d.wrapped).should_receive(
'create_container').with_args(env.workflow.image,
command=["/bin/bash"]).and_return(
{'Id': CONTAINER_ID}))
if remove_fails:
(flexmock(docker_tasker.tasker.d.wrapped).should_receive(
'remove_container').with_args(CONTAINER_ID).and_raise(
Exception('error')))
else:
(flexmock(docker_tasker.tasker.d.wrapped).should_receive(
'remove_container').with_args(CONTAINER_ID))
if has_archive:
(flexmock(docker_tasker.tasker.d.wrapped).should_receive(
'get_archive').with_args(CONTAINER_ID,
'/manifests').and_return(mock_stream, {}))
elif has_archive is not None:
response = Response()
response.status_code = 404
(flexmock(docker_tasker.tasker.d.wrapped).should_receive(
'get_archive').with_args(CONTAINER_ID, '/manifests').and_raise(
NotFound('Not found', response=response)))
else:
response = Response()
response.status_code = 500
(flexmock(docker_tasker.tasker.d.wrapped).should_receive(
'get_archive').with_args(CONTAINER_ID, '/manifests').and_raise(
Exception('error')))
return env.create_runner(docker_tasker)
def get(self, container: str) -> FakeContainer:
result = self.containers.get(container, None)
if result is None:
raise NotFound("Container not found")
return result
def test_get_image_uniq_tag_by_id__image_not_found(self):
self._client.inspect_image.side_effect = NotFound('Image not found')
image_tag = self._helper.get_image_uniq_tag_by_id(self._image_id)
self.assertEqual(self._image_id, image_tag)