def test_import_flow_no_import_flows(self): self.config(engine_mode='serial', group='taskflow_executor') img_factory = mock.MagicMock() executor = taskflow_executor.TaskExecutor(self.context, self.task_repo, self.img_repo, img_factory) self.task_repo.get.return_value = self.task def create_image(*args, **kwargs): kwargs['image_id'] = UUID1 return self.img_factory.new_image(*args, **kwargs) self.img_repo.get.return_value = self.image img_factory.new_image.side_effect = create_image with mock.patch.object(urllib.request, 'urlopen') as umock: content = b"TEST_IMAGE" umock.return_value = six.BytesIO(content) with mock.patch.object(import_flow, "_get_import_flows") as imock: imock.return_value = (x for x in []) executor.begin_processing(self.task.task_id) image_path = os.path.join(self.test_dir, self.image.image_id) tmp_image_path = os.path.join(self.work_dir, "%s.tasks_import" % image_path) self.assertFalse(os.path.exists(tmp_image_path)) self.assertTrue(os.path.exists(image_path)) self.assertEqual(1, umock.call_count) with open(image_path, 'rb') as ifile: self.assertEqual(content, ifile.read())
def test_import_flow_revert_import_to_fs(self): self.config(engine_mode='serial', group='taskflow_executor') img_factory = mock.MagicMock() executor = taskflow_executor.TaskExecutor(self.context, self.task_repo, self.img_repo, img_factory) self.task_repo.get.return_value = self.task def create_image(*args, **kwargs): kwargs['image_id'] = UUID1 return self.img_factory.new_image(*args, **kwargs) self.img_repo.get.return_value = self.image img_factory.new_image.side_effect = create_image with mock.patch.object(script_utils, 'get_image_data_iter') as dmock: dmock.side_effect = RuntimeError with mock.patch.object(import_flow._ImportToFS, 'revert') as rmock: self.assertRaises(RuntimeError, executor.begin_processing, self.task.task_id) self.assertTrue(rmock.called) self.assertIsInstance(rmock.call_args[1]['result'], failure.Failure) image_path = os.path.join(self.test_dir, self.image.image_id) tmp_image_path = os.path.join(self.work_dir, "%s.tasks_import" % image_path) self.assertFalse(os.path.exists(tmp_image_path)) # Note(sabari): The image should not have been uploaded to # the store as the flow failed before ImportToStore Task. self.assertFalse(os.path.exists(image_path))
def test_import_flow_missing_work_dir(self): self.config(engine_mode='serial', group='taskflow_executor') self.config(work_dir=None, group='task') img_factory = mock.MagicMock() executor = taskflow_executor.TaskExecutor(self.context, self.task_repo, self.img_repo, img_factory) self.task_repo.get.return_value = self.task def create_image(*args, **kwargs): kwargs['image_id'] = UUID1 return self.img_factory.new_image(*args, **kwargs) self.img_repo.get.return_value = self.image img_factory.new_image.side_effect = create_image with mock.patch.object(script_utils, 'get_image_data_iter') as dmock: dmock.return_value = six.BytesIO(b"TEST_IMAGE") with mock.patch.object(import_flow._ImportToFS, 'execute') as emk: executor.begin_processing(self.task.task_id) self.assertFalse(emk.called) image_path = os.path.join(self.test_dir, self.image.image_id) tmp_image_path = os.path.join(self.work_dir, "%s.tasks_import" % image_path) self.assertFalse(os.path.exists(tmp_image_path)) self.assertTrue(os.path.exists(image_path))
def test_import_flow(self): self.config(engine_mode='serial', group='taskflow_executor') img_factory = mock.MagicMock() executor = taskflow_executor.TaskExecutor(self.context, self.task_repo, self.img_repo, img_factory) self.task_repo.get.return_value = self.task def create_image(*args, **kwargs): kwargs['image_id'] = UUID1 return self.img_factory.new_image(*args, **kwargs) self.img_repo.get.return_value = self.image img_factory.new_image.side_effect = create_image with mock.patch.object(script_utils, 'get_image_data_iter') as dmock: dmock.return_value = cStringIO("TEST_IMAGE") executor.begin_processing(self.task.task_id) image_path = os.path.join(self.test_dir, self.image.image_id) tmp_image_path = os.path.join(self.work_dir, "%s.tasks_import" % image_path) self.assertFalse(os.path.exists(tmp_image_path)) self.assertTrue(os.path.exists(image_path)) self.assertEqual(1, len(list(self.image.locations))) self.assertEqual( "file://%s/%s" % (self.test_dir, self.image.image_id), self.image.locations[0]['url'])
def test_import_flow_revert(self): self.config(engine_mode='serial', group='taskflow_executor') img_factory = mock.MagicMock() executor = taskflow_executor.TaskExecutor(self.context, self.task_repo, self.img_repo, img_factory) self.task_repo.get.return_value = self.task def create_image(*args, **kwargs): kwargs['image_id'] = UUID1 return self.img_factory.new_image(*args, **kwargs) self.img_repo.get.return_value = self.image img_factory.new_image.side_effect = create_image with mock.patch.object(script_utils, 'get_image_data_iter') as dmock: dmock.return_value = cStringIO("TEST_IMAGE") with mock.patch.object(import_flow, "_get_import_flows") as imock: imock.return_value = (x for x in [_ErrorTask()]) self.assertRaises(RuntimeError, executor.begin_processing, self.task.task_id) image_path = os.path.join(self.test_dir, self.image.image_id) tmp_image_path = os.path.join(self.work_dir, "%s.tasks_import" % image_path) self.assertFalse(os.path.exists(tmp_image_path)) # NOTE(flaper87): Eventually, we want this to be assertTrue. # The current issue is there's no way to tell taskflow to # continue on failures. That is, revert the subflow but keep # executing the parent flow. Under discussion/development. self.assertFalse(os.path.exists(image_path))
def test_import_flow_with_convert_and_introspect(self): self.config(engine_mode='serial', group='taskflow_executor') image = self.img_factory.new_image(image_id=UUID1, disk_format='raw', container_format='bare') img_factory = mock.MagicMock() executor = taskflow_executor.TaskExecutor(self.context, self.task_repo, self.img_repo, img_factory) self.task_repo.get.return_value = self.task def create_image(*args, **kwargs): kwargs['image_id'] = UUID1 return self.img_factory.new_image(*args, **kwargs) self.img_repo.get.return_value = image img_factory.new_image.side_effect = create_image with mock.patch.object(script_utils, 'get_image_data_iter') as dmock: dmock.return_value = StringIO.StringIO("TEST_IMAGE") with mock.patch.object(processutils, 'execute') as exc_mock: result = json.dumps({ "virtual-size": 10737418240, "filename": "/tmp/image.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": 373030912, "format-specific": { "type": "qcow2", "data": { "compat": "0.10" } }, "dirty-flag": False }) # NOTE(flaper87): First result for the conversion step and # the second one for the introspection one. The later *must* # come after the former. If not, the current builtin flow # process will be unsound. # Follow-up work will fix this by having a better way to handle # task's dependencies and activation. exc_mock.side_effect = [("", None), (result, None)] executor.begin_processing(self.task.task_id) image_path = os.path.join(self.test_dir, image.image_id) tmp_image_path = "%s.tasks_import" % image_path self.assertFalse(os.path.exists(tmp_image_path)) self.assertTrue(os.path.exists(image_path)) self.assertEqual('qcow2', image.disk_format) self.assertEqual(10737418240, image.virtual_size)
def test_import_flow_backed_file_import_to_fs(self): self.config(engine_mode='serial', group='taskflow_executor') img_factory = mock.MagicMock() executor = taskflow_executor.TaskExecutor(self.context, self.task_repo, self.img_repo, img_factory) self.task_repo.get.return_value = self.task def create_image(*args, **kwargs): kwargs['image_id'] = UUID1 return self.img_factory.new_image(*args, **kwargs) self.img_repo.get.return_value = self.image img_factory.new_image.side_effect = create_image with mock.patch.object(script_utils, 'get_image_data_iter') as dmock: dmock.return_value = six.BytesIO(b"TEST_IMAGE") with mock.patch.object(putils, 'trycmd') as tmock: tmock.return_value = (json.dumps( {'backing-filename': '/etc/password'}), None) with mock.patch.object(import_flow._ImportToFS, 'revert') as rmock: self.assertRaises(RuntimeError, executor.begin_processing, self.task.task_id) self.assertTrue(rmock.called) self.assertIsInstance(rmock.call_args[1]['result'], failure.Failure) self._assert_qemu_process_limits(tmock) image_path = os.path.join(self.test_dir, self.image.image_id) fname = "%s.tasks_import" % image_path tmp_image_path = os.path.join(self.work_dir, fname) self.assertFalse(os.path.exists(tmp_image_path)) # Note(sabari): The image should not have been uploaded to # the store as the flow failed before ImportToStore Task. self.assertFalse(os.path.exists(image_path))
def setUp(self): super(TestTaskExecutor, self).setUp() glance_store.register_opts(CONF) self.config(default_store='file', stores=['file', 'http'], filesystem_store_datadir=self.test_dir, group="glance_store") glance_store.create_stores(CONF) self.config(engine_mode='serial', group='taskflow_executor') self.context = mock.Mock() self.task_repo = mock.Mock() self.image_repo = mock.Mock() self.image_factory = mock.Mock() task_input = { "import_from": "http://cloud.foo/image.qcow2", "import_from_format": "qcow2", "image_properties": { 'disk_format': 'qcow2', 'container_format': 'bare' } } task_ttl = CONF.task.task_time_to_live self.task_type = 'import' self.task_factory = domain.TaskFactory() self.task = self.task_factory.new_task(self.task_type, TENANT1, task_time_to_live=task_ttl, task_input=task_input) self.executor = taskflow_executor.TaskExecutor(self.context, self.task_repo, self.image_repo, self.image_factory)
def test_import_flow_with_convert_and_introspect(self): self.config(engine_mode='serial', group='taskflow_executor') image = self.img_factory.new_image(image_id=UUID1, disk_format='raw', container_format='bare') img_factory = mock.MagicMock() executor = taskflow_executor.TaskExecutor(self.context, self.task_repo, self.img_repo, img_factory) self.task_repo.get.return_value = self.task def create_image(*args, **kwargs): kwargs['image_id'] = UUID1 return self.img_factory.new_image(*args, **kwargs) self.img_repo.get.return_value = image img_factory.new_image.side_effect = create_image image_path = os.path.join(self.work_dir, image.image_id) def fake_execute(*args, **kwargs): if 'info' in args: # NOTE(flaper87): Make sure the file actually # exists. Extra check to verify previous tasks did # what they were supposed to do. assert os.path.exists(args[3].split("file://")[-1]) return (json.dumps({ "virtual-size": 10737418240, "filename": "/tmp/image.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": 373030912, "format-specific": { "type": "qcow2", "data": { "compat": "0.10" } }, "dirty-flag": False }), None) open("%s.converted" % image_path, 'a').close() return ("", None) with mock.patch.object(script_utils, 'get_image_data_iter') as dmock: dmock.return_value = six.BytesIO("TEST_IMAGE") with mock.patch.object(processutils, 'execute') as exc_mock: exc_mock.side_effect = fake_execute executor.begin_processing(self.task.task_id) # NOTE(flaper87): DeleteFromFS should've deleted this # file. Make sure it doesn't exist. self.assertFalse(os.path.exists(image_path)) # NOTE(flaper87): Workdir should be empty after all # the tasks have been executed. self.assertEqual([], os.listdir(self.work_dir)) self.assertEqual('qcow2', image.disk_format) self.assertEqual(10737418240, image.virtual_size)
def test_import_flow_with_convert_and_introspect(self): self.config(engine_mode='serial', group='taskflow_executor') image = self.img_factory.new_image(image_id=UUID1, disk_format='raw', container_format='bare') img_factory = mock.MagicMock() executor = taskflow_executor.TaskExecutor( self.context, self.task_repo, self.img_repo, img_factory) self.task_repo.get.return_value = self.task def create_image(*args, **kwargs): kwargs['image_id'] = UUID1 return self.img_factory.new_image(*args, **kwargs) self.img_repo.get.return_value = image img_factory.new_image.side_effect = create_image image_path = os.path.join(self.work_dir, image.image_id) def fake_execute(*args, **kwargs): if 'info' in args: # NOTE(flaper87): Make sure the file actually # exists. Extra check to verify previous tasks did # what they were supposed to do. assert os.path.exists(args[3].split("file://")[-1]) return (json.dumps({ "virtual-size": 10737418240, "filename": "/tmp/image.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": 373030912, "format-specific": { "type": "qcow2", "data": { "compat": "0.10" } }, "dirty-flag": False }), None) open("%s.converted" % image_path, 'a').close() return ("", None) with mock.patch.object(script_utils, 'get_image_data_iter') as dmock: dmock.return_value = six.BytesIO(b"TEST_IMAGE") with mock.patch.object(processutils, 'execute') as exc_mock: exc_mock.side_effect = fake_execute executor.begin_processing(self.task.task_id) # NOTE(flaper87): DeleteFromFS should've deleted this # file. Make sure it doesn't exist. self.assertFalse(os.path.exists(image_path)) # NOTE(flaper87): Workdir should be empty after all # the tasks have been executed. self.assertEqual([], os.listdir(self.work_dir)) self.assertEqual('qcow2', image.disk_format) self.assertEqual(10737418240, image.virtual_size) # NOTE(hemanthm): Asserting that the source format is passed # to qemu-utis to avoid inferring the image format when # converting. This shields us from an attack vector described # at https://bugs.launchpad.net/glance/+bug/1449062/comments/72 # # A total of three calls will be made to 'execute': 'info', # 'convert' and 'info' towards introspection, conversion and # OVF packaging respectively. We care about the 'convert' call # here, hence we fetch the 2nd set of args from the args list. convert_call_args, _ = exc_mock.call_args_list[1] self.assertIn('-f', convert_call_args)