def test_import_flow_no_import_flows(self):
self.config(engine_mode='serial', group='taskflow_executor')
img_factory = mock.MagicMock()
executor = taskflow_executor.TaskExecutor(self.context, self.task_repo,
self.img_repo, img_factory)
self.task_repo.get.return_value = self.task
def create_image(*args, **kwargs):
kwargs['image_id'] = UUID1
return self.img_factory.new_image(*args, **kwargs)
self.img_repo.get.return_value = self.image
img_factory.new_image.side_effect = create_image
with mock.patch.object(urllib.request, 'urlopen') as umock:
content = b"TEST_IMAGE"
umock.return_value = six.BytesIO(content)
with mock.patch.object(import_flow, "_get_import_flows") as imock:
imock.return_value = (x for x in [])
executor.begin_processing(self.task.task_id)
image_path = os.path.join(self.test_dir, self.image.image_id)
tmp_image_path = os.path.join(self.work_dir,
"%s.tasks_import" % image_path)
self.assertFalse(os.path.exists(tmp_image_path))
self.assertTrue(os.path.exists(image_path))
self.assertEqual(1, umock.call_count)
with open(image_path, 'rb') as ifile:
self.assertEqual(content, ifile.read())
def test_import_flow_revert_import_to_fs(self):
self.config(engine_mode='serial', group='taskflow_executor')
img_factory = mock.MagicMock()
executor = taskflow_executor.TaskExecutor(self.context, self.task_repo,
self.img_repo, img_factory)
self.task_repo.get.return_value = self.task
def create_image(*args, **kwargs):
kwargs['image_id'] = UUID1
return self.img_factory.new_image(*args, **kwargs)
self.img_repo.get.return_value = self.image
img_factory.new_image.side_effect = create_image
with mock.patch.object(script_utils, 'get_image_data_iter') as dmock:
dmock.side_effect = RuntimeError
with mock.patch.object(import_flow._ImportToFS, 'revert') as rmock:
self.assertRaises(RuntimeError, executor.begin_processing,
self.task.task_id)
self.assertTrue(rmock.called)
self.assertIsInstance(rmock.call_args[1]['result'],
failure.Failure)
image_path = os.path.join(self.test_dir, self.image.image_id)
tmp_image_path = os.path.join(self.work_dir,
"%s.tasks_import" % image_path)
self.assertFalse(os.path.exists(tmp_image_path))
# Note(sabari): The image should not have been uploaded to
# the store as the flow failed before ImportToStore Task.
self.assertFalse(os.path.exists(image_path))
def test_import_flow_missing_work_dir(self):
self.config(engine_mode='serial', group='taskflow_executor')
self.config(work_dir=None, group='task')
img_factory = mock.MagicMock()
executor = taskflow_executor.TaskExecutor(self.context, self.task_repo,
self.img_repo, img_factory)
self.task_repo.get.return_value = self.task
def create_image(*args, **kwargs):
kwargs['image_id'] = UUID1
return self.img_factory.new_image(*args, **kwargs)
self.img_repo.get.return_value = self.image
img_factory.new_image.side_effect = create_image
with mock.patch.object(script_utils, 'get_image_data_iter') as dmock:
dmock.return_value = six.BytesIO(b"TEST_IMAGE")
with mock.patch.object(import_flow._ImportToFS, 'execute') as emk:
executor.begin_processing(self.task.task_id)
self.assertFalse(emk.called)
image_path = os.path.join(self.test_dir, self.image.image_id)
tmp_image_path = os.path.join(self.work_dir,
"%s.tasks_import" % image_path)
self.assertFalse(os.path.exists(tmp_image_path))
self.assertTrue(os.path.exists(image_path))
def test_import_flow(self):
self.config(engine_mode='serial', group='taskflow_executor')
img_factory = mock.MagicMock()
executor = taskflow_executor.TaskExecutor(self.context, self.task_repo,
self.img_repo, img_factory)
self.task_repo.get.return_value = self.task
def create_image(*args, **kwargs):
kwargs['image_id'] = UUID1
return self.img_factory.new_image(*args, **kwargs)
self.img_repo.get.return_value = self.image
img_factory.new_image.side_effect = create_image
with mock.patch.object(script_utils, 'get_image_data_iter') as dmock:
dmock.return_value = cStringIO("TEST_IMAGE")
executor.begin_processing(self.task.task_id)
image_path = os.path.join(self.test_dir, self.image.image_id)
tmp_image_path = os.path.join(self.work_dir,
"%s.tasks_import" % image_path)
self.assertFalse(os.path.exists(tmp_image_path))
self.assertTrue(os.path.exists(image_path))
self.assertEqual(1, len(list(self.image.locations)))
self.assertEqual(
"file://%s/%s" % (self.test_dir, self.image.image_id),
self.image.locations[0]['url'])
def test_import_flow_revert(self):
self.config(engine_mode='serial', group='taskflow_executor')
img_factory = mock.MagicMock()
executor = taskflow_executor.TaskExecutor(self.context, self.task_repo,
self.img_repo, img_factory)
self.task_repo.get.return_value = self.task
def create_image(*args, **kwargs):
kwargs['image_id'] = UUID1
return self.img_factory.new_image(*args, **kwargs)
self.img_repo.get.return_value = self.image
img_factory.new_image.side_effect = create_image
with mock.patch.object(script_utils, 'get_image_data_iter') as dmock:
dmock.return_value = cStringIO("TEST_IMAGE")
with mock.patch.object(import_flow, "_get_import_flows") as imock:
imock.return_value = (x for x in [_ErrorTask()])
self.assertRaises(RuntimeError, executor.begin_processing,
self.task.task_id)
image_path = os.path.join(self.test_dir, self.image.image_id)
tmp_image_path = os.path.join(self.work_dir,
"%s.tasks_import" % image_path)
self.assertFalse(os.path.exists(tmp_image_path))
# NOTE(flaper87): Eventually, we want this to be assertTrue.
# The current issue is there's no way to tell taskflow to
# continue on failures. That is, revert the subflow but keep
# executing the parent flow. Under discussion/development.
self.assertFalse(os.path.exists(image_path))
def test_import_flow_with_convert_and_introspect(self):
self.config(engine_mode='serial', group='taskflow_executor')
image = self.img_factory.new_image(image_id=UUID1,
disk_format='raw',
container_format='bare')
img_factory = mock.MagicMock()
executor = taskflow_executor.TaskExecutor(self.context, self.task_repo,
self.img_repo, img_factory)
self.task_repo.get.return_value = self.task
def create_image(*args, **kwargs):
kwargs['image_id'] = UUID1
return self.img_factory.new_image(*args, **kwargs)
self.img_repo.get.return_value = image
img_factory.new_image.side_effect = create_image
with mock.patch.object(script_utils, 'get_image_data_iter') as dmock:
dmock.return_value = StringIO.StringIO("TEST_IMAGE")
with mock.patch.object(processutils, 'execute') as exc_mock:
result = json.dumps({
"virtual-size": 10737418240,
"filename": "/tmp/image.qcow2",
"cluster-size": 65536,
"format": "qcow2",
"actual-size": 373030912,
"format-specific": {
"type": "qcow2",
"data": {
"compat": "0.10"
}
},
"dirty-flag": False
})
# NOTE(flaper87): First result for the conversion step and
# the second one for the introspection one. The later *must*
# come after the former. If not, the current builtin flow
# process will be unsound.
# Follow-up work will fix this by having a better way to handle
# task's dependencies and activation.
exc_mock.side_effect = [("", None), (result, None)]
executor.begin_processing(self.task.task_id)
image_path = os.path.join(self.test_dir, image.image_id)
tmp_image_path = "%s.tasks_import" % image_path
self.assertFalse(os.path.exists(tmp_image_path))
self.assertTrue(os.path.exists(image_path))
self.assertEqual('qcow2', image.disk_format)
self.assertEqual(10737418240, image.virtual_size)
def test_import_flow_backed_file_import_to_fs(self):
self.config(engine_mode='serial', group='taskflow_executor')
img_factory = mock.MagicMock()
executor = taskflow_executor.TaskExecutor(self.context, self.task_repo,
self.img_repo, img_factory)
self.task_repo.get.return_value = self.task
def create_image(*args, **kwargs):
kwargs['image_id'] = UUID1
return self.img_factory.new_image(*args, **kwargs)
self.img_repo.get.return_value = self.image
img_factory.new_image.side_effect = create_image
with mock.patch.object(script_utils, 'get_image_data_iter') as dmock:
dmock.return_value = six.BytesIO(b"TEST_IMAGE")
with mock.patch.object(putils, 'trycmd') as tmock:
tmock.return_value = (json.dumps(
{'backing-filename': '/etc/password'}), None)
with mock.patch.object(import_flow._ImportToFS,
'revert') as rmock:
self.assertRaises(RuntimeError, executor.begin_processing,
self.task.task_id)
self.assertTrue(rmock.called)
self.assertIsInstance(rmock.call_args[1]['result'],
failure.Failure)
self._assert_qemu_process_limits(tmock)
image_path = os.path.join(self.test_dir,
self.image.image_id)
fname = "%s.tasks_import" % image_path
tmp_image_path = os.path.join(self.work_dir, fname)
self.assertFalse(os.path.exists(tmp_image_path))
# Note(sabari): The image should not have been uploaded to
# the store as the flow failed before ImportToStore Task.
self.assertFalse(os.path.exists(image_path))
def setUp(self):
super(TestTaskExecutor, self).setUp()
glance_store.register_opts(CONF)
self.config(default_store='file',
stores=['file', 'http'],
filesystem_store_datadir=self.test_dir,
group="glance_store")
glance_store.create_stores(CONF)
self.config(engine_mode='serial', group='taskflow_executor')
self.context = mock.Mock()
self.task_repo = mock.Mock()
self.image_repo = mock.Mock()
self.image_factory = mock.Mock()
task_input = {
"import_from": "http://cloud.foo/image.qcow2",
"import_from_format": "qcow2",
"image_properties": {
'disk_format': 'qcow2',
'container_format': 'bare'
}
}
task_ttl = CONF.task.task_time_to_live
self.task_type = 'import'
self.task_factory = domain.TaskFactory()
self.task = self.task_factory.new_task(self.task_type,
TENANT1,
task_time_to_live=task_ttl,
task_input=task_input)
self.executor = taskflow_executor.TaskExecutor(self.context,
self.task_repo,
self.image_repo,
self.image_factory)
def test_import_flow_with_convert_and_introspect(self):
self.config(engine_mode='serial', group='taskflow_executor')
image = self.img_factory.new_image(image_id=UUID1,
disk_format='raw',
container_format='bare')
img_factory = mock.MagicMock()
executor = taskflow_executor.TaskExecutor(self.context, self.task_repo,
self.img_repo, img_factory)
self.task_repo.get.return_value = self.task
def create_image(*args, **kwargs):
kwargs['image_id'] = UUID1
return self.img_factory.new_image(*args, **kwargs)
self.img_repo.get.return_value = image
img_factory.new_image.side_effect = create_image
image_path = os.path.join(self.work_dir, image.image_id)
def fake_execute(*args, **kwargs):
if 'info' in args:
# NOTE(flaper87): Make sure the file actually
# exists. Extra check to verify previous tasks did
# what they were supposed to do.
assert os.path.exists(args[3].split("file://")[-1])
return (json.dumps({
"virtual-size": 10737418240,
"filename": "/tmp/image.qcow2",
"cluster-size": 65536,
"format": "qcow2",
"actual-size": 373030912,
"format-specific": {
"type": "qcow2",
"data": {
"compat": "0.10"
}
},
"dirty-flag": False
}), None)
open("%s.converted" % image_path, 'a').close()
return ("", None)
with mock.patch.object(script_utils, 'get_image_data_iter') as dmock:
dmock.return_value = six.BytesIO("TEST_IMAGE")
with mock.patch.object(processutils, 'execute') as exc_mock:
exc_mock.side_effect = fake_execute
executor.begin_processing(self.task.task_id)
# NOTE(flaper87): DeleteFromFS should've deleted this
# file. Make sure it doesn't exist.
self.assertFalse(os.path.exists(image_path))
# NOTE(flaper87): Workdir should be empty after all
# the tasks have been executed.
self.assertEqual([], os.listdir(self.work_dir))
self.assertEqual('qcow2', image.disk_format)
self.assertEqual(10737418240, image.virtual_size)
def test_import_flow_with_convert_and_introspect(self):
self.config(engine_mode='serial',
group='taskflow_executor')
image = self.img_factory.new_image(image_id=UUID1,
disk_format='raw',
container_format='bare')
img_factory = mock.MagicMock()
executor = taskflow_executor.TaskExecutor(
self.context,
self.task_repo,
self.img_repo,
img_factory)
self.task_repo.get.return_value = self.task
def create_image(*args, **kwargs):
kwargs['image_id'] = UUID1
return self.img_factory.new_image(*args, **kwargs)
self.img_repo.get.return_value = image
img_factory.new_image.side_effect = create_image
image_path = os.path.join(self.work_dir, image.image_id)
def fake_execute(*args, **kwargs):
if 'info' in args:
# NOTE(flaper87): Make sure the file actually
# exists. Extra check to verify previous tasks did
# what they were supposed to do.
assert os.path.exists(args[3].split("file://")[-1])
return (json.dumps({
"virtual-size": 10737418240,
"filename": "/tmp/image.qcow2",
"cluster-size": 65536,
"format": "qcow2",
"actual-size": 373030912,
"format-specific": {
"type": "qcow2",
"data": {
"compat": "0.10"
}
},
"dirty-flag": False
}), None)
open("%s.converted" % image_path, 'a').close()
return ("", None)
with mock.patch.object(script_utils, 'get_image_data_iter') as dmock:
dmock.return_value = six.BytesIO(b"TEST_IMAGE")
with mock.patch.object(processutils, 'execute') as exc_mock:
exc_mock.side_effect = fake_execute
executor.begin_processing(self.task.task_id)
# NOTE(flaper87): DeleteFromFS should've deleted this
# file. Make sure it doesn't exist.
self.assertFalse(os.path.exists(image_path))
# NOTE(flaper87): Workdir should be empty after all
# the tasks have been executed.
self.assertEqual([], os.listdir(self.work_dir))
self.assertEqual('qcow2', image.disk_format)
self.assertEqual(10737418240, image.virtual_size)
# NOTE(hemanthm): Asserting that the source format is passed
# to qemu-utis to avoid inferring the image format when
# converting. This shields us from an attack vector described
# at https://bugs.launchpad.net/glance/+bug/1449062/comments/72
#
# A total of three calls will be made to 'execute': 'info',
# 'convert' and 'info' towards introspection, conversion and
# OVF packaging respectively. We care about the 'convert' call
# here, hence we fetch the 2nd set of args from the args list.
convert_call_args, _ = exc_mock.call_args_list[1]
self.assertIn('-f', convert_call_args)
