def test_pull_retry(self): workflow = self.workflow() tasker = MockerTasker() expected_pullspec = '%s/%s' % (self.CRANE_URI, self.TEST_UNIQUE_IMAGE) test_id = 'sha256:(new)' (flexmock(tasker).should_call('pull_image').and_return( expected_pullspec).times(3)) (flexmock(tasker).should_receive('inspect_image').with_args( expected_pullspec).and_raise( NotFound('message', flexmock(content=None))).and_raise( NotFound('message', flexmock(content=None))).and_return({ 'Id': test_id }).times(3)) plugin = PulpPullPlugin(tasker, workflow, timeout=1, retry_delay=0.6) # Plugin return value is the new ID assert plugin.run() == test_id assert len(tasker.pulled_images) == 3 for image in tasker.pulled_images: pulled = image.to_str() assert pulled == expected_pullspec # Image ID is updated in workflow assert workflow.builder.image_id == test_id
def test_down_with_no_resources(self): project = Project.from_config( name='test', client=self.mock_client, config_data=Config( project_name='', version=V2_0, services=[{ 'name': 'web', 'image': BUSYBOX_IMAGE_WITH_TAG, }], networks={'default': {}}, volumes={'data': {}}, secrets=None, configs=None, ), ) self.mock_client.remove_network.side_effect = NotFound( None, None, 'oops') self.mock_client.remove_volume.side_effect = NotFound( None, None, 'oops') project.down(ImageType.all, True) self.mock_client.remove_image.assert_called_once_with( BUSYBOX_IMAGE_WITH_TAG)
def test_ensure_socat_running_some_wait(mocker, socat_mocked): mocker.patch.object(socat_mocked.client.containers, 'get').side_effect = [ NotFound('not found'), NotFound('not found'), mocker.MagicMock(status='creating'), mocker.MagicMock(status='creating'), mocker.MagicMock(status='creating'), mocker.MagicMock(status='running') ] # noinspection PyProtectedMember socat_mocked._ensure_socat_running() # noinspection PyUnresolvedReferences assert socat_mocked.sleep.call_count == 5
def unregister_container(instance: object, name: str) -> None: if name not in instance._containers: raise NotFound( f"{instance}: there is no container registered for name `{name}'" ) del instance._containers[name] LOGGER.debug("%s: container `%s' unregistered", instance, name)
def run_couchdb_docker_container(docker_client): # if there is no image we pull it try: couchdb_image = docker_client.images.get("couchdb") except NotFound as e: couchdb_image = docker_client.images.pull("couchdb", tag="latest") assert couchdb_image is not None # if container is not running we will start it try: couch_container = docker_client.containers.get("appguard_couch") if couch_container.status == "exited" or (couch_container.status == "created"): couch_container.remove() raise NotFound("Container Exited or could not be started") except NotFound as e: print("Couch docker container not found or not running\n") couch_container = docker_client.containers.run( "couchdb", name="appguard_couch", ports={"5984/tcp": 5984}, detach=True) time.sleep(5) assert couch_container.status == "running" or ( couch_container.status == "created" and not couch_container.attrs["State"]["Error"]) return couch_container
def compose_project_by_name(project_name: str) -> ComposeProject: project_config: ComposeProjectConfig = compose_config() if project_config.project_name == project_name: return ComposeProject(project_name=project_name, project_config=project_config) else: message = f"couldn't find project '{project_name}'" raise NotFound(message="Not Found", explanation=message)
def mock_env(tmpdir, docker_tasker, has_appregistry_label=False, appregistry_label=False, has_bundle_label=True, bundle_label=True, has_archive=True, scratch=False, orchestrator=False, selected_platform=True, empty_archive=False, remove_fails=False): build_json = {'metadata': {'labels': {'scratch': scratch}}} flexmock(util).should_receive('get_build_json').and_return(build_json) mock_dockerfile( tmpdir, has_appregistry_label=has_appregistry_label, appregistry_label=appregistry_label, has_bundle_label=has_bundle_label, bundle_label=bundle_label ) workflow = mock_workflow(tmpdir, for_orchestrator=orchestrator) mock_stream = generate_archive(tmpdir, empty_archive) plugin_conf = [{'name': ExportOperatorManifestsPlugin.key}] if selected_platform: plugin_conf[0]['args'] = {'operator_manifests_extract_platform': machine(), 'platform': machine()} runner = PostBuildPluginsRunner(docker_tasker, workflow, plugin_conf) (flexmock(docker_tasker.tasker.d.wrapped) .should_receive('create_container') .with_args(workflow.image, command=["/bin/bash"]) .and_return({'Id': CONTAINER_ID})) if remove_fails: (flexmock(docker_tasker.tasker.d.wrapped) .should_receive('remove_container') .with_args(CONTAINER_ID) .and_raise(Exception('error'))) else: (flexmock(docker_tasker.tasker.d.wrapped) .should_receive('remove_container') .with_args(CONTAINER_ID)) if has_archive: (flexmock(docker_tasker.tasker.d.wrapped) .should_receive('get_archive') .with_args(CONTAINER_ID, '/manifests') .and_return(mock_stream, {})) elif has_archive is not None: response = Response() response.status_code = 404 (flexmock(docker_tasker.tasker.d.wrapped) .should_receive('get_archive') .with_args(CONTAINER_ID, '/manifests') .and_raise(NotFound('Not found', response=response))) else: response = Response() response.status_code = 500 (flexmock(docker_tasker.tasker.d.wrapped) .should_receive('get_archive') .with_args(CONTAINER_ID, '/manifests') .and_raise(Exception('error'))) return runner
def __call__(self, cmd, timeout=10): deprecation("consider to use Docker Python module instead of using Docker CLI commands") res = LOCALRUNNER.run('docker {}'.format(cmd), ignore_status=True, timeout=timeout) if res.exit_status: if 'No such container:' in res.stderr: raise NotFound(res.stderr) raise DockerException('command: {}, error: {}, output: {}'.format(cmd, res.stderr, res.stdout)) return res
def get_container(instance: object, name, raise_not_found_exc: bool = True) -> Optional[Container]: assert name is not None, "None is not allowed as a container name" container = instance._containers.get(name) if container is None and raise_not_found_exc: raise NotFound(f"There is no container `{name}' for {instance}.") if container: container.reload() return container
def test_down_with_no_resources(self): project = Project.from_config( name='test', client=self.mock_client, config_data=Config( version='2', services=[{ 'name': 'web', 'image': 'busybox:latest', }], networks={'default': {}}, volumes={'data': {}}, ), ) self.mock_client.remove_network.side_effect = NotFound(None, None, 'oops') self.mock_client.remove_volume.side_effect = NotFound(None, None, 'oops') project.down(ImageType.all, True) self.mock_client.remove_image.assert_called_once_with("busybox:latest")
def compose_service(project_name: str, service_name: str) -> ComposeService: try: service_config: ComposeServiceConfig = compose_config( ).get_service_config(project_name=project_name, service_name=service_name) return ComposeService(project_name=project_name, service_name=service_name, service_config=service_config) except StopIteration: message = f"couldn't find service '{service_name}' in project '{project_name}'" raise NotFound(message="Not Found", explanation=message)
def test_must_work_when_container_is_not_found(self): self.container.is_created.return_value = True real_container_mock = Mock() self.mock_docker_client.containers.get.side_effect = NotFound("msg") real_container_mock.remove = Mock() self.container.stop() self.mock_docker_client.containers.get.assert_called_with("someid") real_container_mock.remove.assert_not_called() # Ensure ID remains set self.assertIsNotNone(self.container.id)
def test_check_and_download_remote_plugin_not_installed(mock_setting_get_instance, mock_get_xtables_lock_mount, mock_configure_xtables_mount, docker_plugin, mock_plugin, mock_setting): mock_setting.remote_url = "http://remote-url.kt" mock_setting_get_instance.return_value = mock_setting docker_plugin.client.plugins.get.return_value = None docker_plugin.client.plugins.get.side_effect = NotFound('Fail') with pytest.raises(Exception) as e: docker_plugin.check_and_download_plugin() assert str(e.value) == "Kathara Network Plugin not found on remote Docker connection." assert not mock_get_xtables_lock_mount.called assert not mock_configure_xtables_mount.called
def test_check_and_download_plugin_not_found(mock_setting_get_instance, mock_get_xtables_lock_mount, docker_plugin, mock_plugin, mock_setting): mock_setting_get_instance.return_value = mock_setting docker_plugin.client.plugins.get.return_value = None docker_plugin.client.plugins.get.side_effect = NotFound('Fail') docker_plugin.client.plugins.install.return_value = mock_plugin mock_plugin.enabled = False docker_plugin.check_and_download_plugin() docker_plugin.client.plugins.get.assert_called_once_with("kathara/katharanp:" + utils.get_architecture()) assert not mock_plugin.upgrade.called mock_get_xtables_lock_mount.assert_called_once() mock_plugin.enable.assert_called_once() assert not mock_plugin.disable.called
def test_must_work_when_container_is_not_found(self): self.container.is_created.return_value = True real_container_mock = Mock() self.mock_docker_client.containers.get.side_effect = NotFound("msg") real_container_mock.remove = Mock() self.container.delete() self.mock_docker_client.containers.get.assert_called_with("someid") real_container_mock.remove.assert_not_called() # Must reset ID to None because container is now gone self.assertIsNone(self.container.id)
def test_pull_timeout(self): workflow = self.workflow() tasker = MockerTasker() (flexmock(tasker).should_call('pull_image').and_return( self.EXPECTED_PULLSPEC).times(3)) (flexmock(tasker).should_receive('inspect_image').with_args( self.EXPECTED_PULLSPEC).and_raise( NotFound('message', flexmock(content=None))).times(3)) plugin = PulpPullPlugin(tasker, workflow, timeout=1, retry_delay=0.6) # Should raise a timeout exception with pytest.raises(CraneTimeoutError): plugin.run()
def parse_inspect_output(out, item_type): """ Parses the output of the Docker CLI 'docker inspect <container>' or 'docker network inspect <network>'. Essentially just returns the parsed JSON string, like the Docker API does. :param out: CLI output. :type out: unicode | str :param item_type: Type of the item that has been inspected (e.g. 'container'). :type item_type: unicode | str :return: Parsed result. :rtype: dict """ parsed = json.loads(out, encoding='utf-8') if parsed: return parsed[0] raise NotFound("{0} not found.".format(item_type.title()), None)
def test_pull_timeout(self): workflow = self.workflow() tasker = MockerTasker() expected_pullspec = '%s/%s' % (self.CRANE_URI, self.TEST_UNIQUE_IMAGE) (flexmock(tasker).should_call('pull_image').and_return( expected_pullspec).times(3)) (flexmock(tasker).should_receive('inspect_image').with_args( expected_pullspec).and_raise( NotFound('message', flexmock(content=None))).times(3)) plugin = PulpPullPlugin(tasker, workflow, timeout=1, retry_delay=0.6) # Should raise a timeout exception with pytest.raises(CraneTimeoutError): plugin.run()
def get_container(cid): if cid == 'eeeee': raise NotFound(None, None, "oops") if cid == 'abcde': name = 'web' labels = {LABEL_SERVICE: name} elif cid == 'ababa': name = 'db' labels = {LABEL_SERVICE: name} else: labels = {} name = '' return { 'Id': cid, 'Config': {'Labels': labels}, 'Name': '/project_%s_1' % name, }
def run_opa_docker_container(docker_client): """ We will use this function for system testing :param docker_client: Docker client handle :type docker_client: :return: docker container handle :rtype: string """ # if there is no image we pull it try: opa_image = docker_client.images.get("openpolicyagent/opa") except NotFound as e: opa_image = docker_client.images.pull("openpolicyagent/opa", tag="latest") assert opa_image is not None # if container is not running we will start it try: # TODO: change name to something more appropriate opa_container = docker_client.containers.get("appguard_opa") if opa_container.status == "exited" or (opa_container.status == "created"): opa_container.remove() raise NotFound("Container Exited or could not be started") except NotFound as e: print("OPA docker container not found or not running. \n") opa_container = docker_client.containers.run( "openpolicyagent/opa", command="run --server --log-level=debug", name="appguard_opa", ports={"8181/tcp": 8181}, detach=True) time.sleep(5) assert opa_container.status == "running" or ( opa_container.status == "created" and not opa_container.attrs["State"]["Error"]) return opa_container
class JobModelTest(TestCase): def setUp(self): self.project = models.Project.objects.create( name='abdallah', url='https://github.com/ZuluPro/abdallah.git') self.build = models.Build.objects.create(project=self.project) def test_create(self): job = models.Job.objects.create(build=self.build) self.assertEqual(job.number, 1) def test_get_absolute_url(self): job = models.Job.objects.create(build=self.build) url = job.get_absolute_url() self.assertIsInstance(url, six.string_types) def test_get_css_class(self): job = models.Job.objects.create(build=self.build) suffix = job.get_css_class() self.assertIsInstance(suffix, six.string_types) def test_container_name(self): job = models.Job.objects.create(build=self.build) self.assertIsInstance(job.container_name, six.string_types) @mock.patch('docker.Client.logs', return_value='FOO-BAR') def test_logs(self, *args): job = models.Job.objects.create(build=self.build) logs = job.logs self.assertIsInstance(logs, six.string_types) self.assertEqual(logs, 'FOO-BAR') @mock.patch('docker.Client.logs', side_effect=NotFound('foo', mock.MagicMock())) def test_logs_not_found_container(self, *args): job = models.Job.objects.create(build=self.build) logs = job.logs self.assertIsInstance(logs, six.string_types)
def raises_NotFound(name: str): raise NotFound("")
def main(args): #: setup parser ----------------------------------------------------------- parser = argparse.ArgumentParser(description='Magen IO Server', usage=("\npython3 server.py " "--csrf" "--clean-init" "--ingestion-data-dir" "\n\nnote:\n" "root privileges are required ")) if inside_docker(): ingestion_data_dir = os.path.join("/opt", "data") else: home_dir = str(Path.home()) ingestion_data_dir = os.path.join(home_dir, "magen_data", "ingestion") parser.add_argument('--ingestion-data-dir', default=ingestion_data_dir, help='Set directory for data files' 'Default is %s' % ingestion_data_dir) parser.add_argument('--clean-init', action='store_false', help='Clean All data when initializing' 'Default is to clean)') parser.add_argument('--csrf', action='store_true', help='Enable Cross Request Forgery protection' 'Default is to not use it)') parser.add_argument('--test', action='store_true', help='Run server in test mode. Used for unit tests' 'Default is to run in production mode)') #: parse CMD arguments ---------------------------------------------------- # args = parser.parse_args() args, _ = parser.parse_known_args(args) """ Main Magen.io Sever """ home_dir = str(Path.home()) ingestion_globals = IngestionGlobals() ingestion_globals.data_dir = args.ingestion_data_dir try: os.makedirs(ingestion_globals.data_dir) except OSError as e: if e.errno != errno.EEXIST: raise # Starting OPA server docker_client = docker.from_env() # if there is no image we pull it try: opa_image = docker_client.images.get("openpolicyagent/opa") except NotFound as e: opa_image = docker_client.images.pull("openpolicyagent/opa", tag="latest") assert opa_image is not None # if container is not running we will start it try: opa_container = docker_client.containers.get("magen_opa") if opa_container.status == "exited" or opa_container.status == "created": opa_container.remove() raise NotFound("Container Exited or could not be started") except NotFound as e: print("OPA docker container not found or not running, starting... \n") opa_container = docker_client.containers.run( "openpolicyagent/opa", command="run --server --log-level=debug", name="magen_opa", ports={"8181/tcp": 8181}, detach=True) time.sleep(5) assert opa_container.status == "running" or ( opa_container.status == "created" and not opa_container.attrs["State"]["Error"]) ks_args = [[]] ks_process = Process(target=ks_server.main, args=ks_args) ks_process.start() time.sleep(2) serverurls = ServerUrls() ks_check_url = serverurls.key_server_base_url + "check/" get_resp_obj = RestClientApis.http_get_and_check_success(ks_check_url) assert get_resp_obj.success is True mongo_ip, mongo_port = mongo_host_port() # We initialize at runtime everything about Mongo and its functions # Any client of the API can change it later db = MainDb.get_instance() db.core_database = MongoCore.get_instance() db.core_database.utils_strategy = MongoUtils.get_instance() db.core_database.asset_strategy = MongoAsset.get_instance() db.core_database.db_ip_port = '{ip}:{port}'.format(ip=mongo_ip, port=mongo_port) db.core_database.utils_strategy.check_db(db.core_database.db_ip_port) db.core_database.initialize() if args.clean_init: success, _ = AssetDbApi.delete_all() assert success is True user_api.drop_user_collection() GridFsApi.delete_all() if args.csrf: app.config['WTF_CSRF_ENABLED'] = True app.config['WTF_CSRF_SECRET_KEY'] = token_hex(16) CSRFProtect(app) else: app.config['WTF_CSRF_ENABLED'] = False app.register_blueprint(main_bp) app.register_blueprint(users_bp) app.register_blueprint(ingestion_file_upload_bp, url_prefix='/magen/ingestion/v2') app.register_blueprint(ingestion_bp_v2, url_prefix='/magen/ingestion/v2') app.register_blueprint(ingestion_bp, url_prefix='/magen/ingestion/v2') app.run('0.0.0.0', 5005, threaded=True)
def _container_not_exists(name): raise NotFound(f'container: {name} not exists')
def mock_env(tmpdir, docker_tasker, has_appregistry_label=False, appregistry_label=False, has_bundle_label=True, bundle_label=True, has_archive=True, scratch=False, orchestrator=False, selected_platform=True, empty_archive=False, remove_fails=False, change_csv_content=False, multiple_csv=False): repo_dir = tmpdir.join('test-operator').mkdir() mock_dockerfile(repo_dir, has_appregistry_label=has_appregistry_label, appregistry_label=appregistry_label, has_bundle_label=has_bundle_label, bundle_label=bundle_label) manifests_dir = mock_manifests_dir(repo_dir) env = (MockEnv().for_plugin( 'postbuild', ExportOperatorManifestsPlugin.key).set_scratch(scratch)) if orchestrator: env.make_orchestrator() class MockSource(object): @property def manifests_dir(self): return manifests_dir # Set a new source object, only the manifests_dir property is required for tests. setattr(env.workflow, 'source', MockSource()) env.workflow.builder.set_df_path(str(repo_dir)) mock_stream = generate_archive(tmpdir, empty_archive, change_csv_content, multiple_csv) if selected_platform: env.set_plugin_args({ 'operator_manifests_extract_platform': machine(), 'platform': machine() }) (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'create_container').with_args(env.workflow.image, command=["/bin/bash"]).and_return( {'Id': CONTAINER_ID})) if remove_fails: (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'remove_container').with_args(CONTAINER_ID).and_raise( Exception('error'))) else: (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'remove_container').with_args(CONTAINER_ID)) if has_archive: (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'get_archive').with_args(CONTAINER_ID, '/manifests').and_return(mock_stream, {})) elif has_archive is not None: response = Response() response.status_code = 404 (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'get_archive').with_args(CONTAINER_ID, '/manifests').and_raise( NotFound('Not found', response=response))) else: response = Response() response.status_code = 500 (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'get_archive').with_args(CONTAINER_ID, '/manifests').and_raise( Exception('error'))) return env.create_runner(docker_tasker)
def get(self, container: str) -> FakeContainer: result = self.containers.get(container, None) if result is None: raise NotFound("Container not found") return result
def test_get_image_uniq_tag_by_id__image_not_found(self): self._client.inspect_image.side_effect = NotFound('Image not found') image_tag = self._helper.get_image_uniq_tag_by_id(self._image_id) self.assertEqual(self._image_id, image_tag)