def execute(self, file_path=None):
"""Bringing the imported image to back end store
:param image_id: Glance Image ID
:param file_path: path to the image file
"""
# NOTE(flaper87): Let's dance... and fall
#
# Unfortunatelly, because of the way our domain layers work and
# the checks done in the FS store, we can't simply rename the file
# and set the location. To do that, we'd have to duplicate the logic
# of every and each of the domain factories (quota, location, etc)
# and we'd also need to hack the FS store to prevent it from raising
# a "duplication path" error. I'd rather have this task copying the
# image bits one more time than duplicating all that logic.
#
# Since I don't think this should be the definitive solution, I'm
# leaving the code below as a reference for what should happen here
# once the FS store and domain code will be able to handle this case.
#
# if file_path is None:
# image_import.set_image_data(image, self.uri, None)
# return
# NOTE(flaper87): Don't assume the image was stored in the
# work_dir. Think in the case this path was provided by another task.
# Also, lets try to neither assume things nor create "logic"
# dependencies between this task and `_ImportToFS`
#
# base_path = os.path.dirname(file_path.split("file://")[-1])
# NOTE(flaper87): Hopefully just scenarios #3 and #4. I say
# hopefully because nothing prevents the user to use the same
# FS store path as a work dir
#
# image_path = os.path.join(base_path, image_id)
#
# if (base_path == CONF.glance_store.filesystem_store_datadir or
# base_path in CONF.glance_store.filesystem_store_datadirs):
# os.rename(file_path, image_path)
#
# image_import.set_image_data(image, image_path, None)
# NOTE(jokke): The different options here are kind of pointless as we
# will need the file path anyways for our delete workflow for now.
# For future proofing keeping this as is.
image = self.image_repo.get(self.image_id)
image_import.set_image_data(image,
file_path or self.uri,
self.task_id,
backend=self.backend)
# NOTE(flaper87): We need to save the image again after the locations
# have been set in the image.
self.image_repo.save(image)
def execute(self, file_path=None):
"""Bringing the imported image to back end store
:param image_id: Glance Image ID
:param file_path: path to the image file
"""
# NOTE(flaper87): Let's dance... and fall
#
# Unfortunatelly, because of the way our domain layers work and
# the checks done in the FS store, we can't simply rename the file
# and set the location. To do that, we'd have to duplicate the logic
# of every and each of the domain factories (quota, location, etc)
# and we'd also need to hack the FS store to prevent it from raising
# a "duplication path" error. I'd rather have this task copying the
# image bits one more time than duplicating all that logic.
#
# Since I don't think this should be the definitive solution, I'm
# leaving the code below as a reference for what should happen here
# once the FS store and domain code will be able to handle this case.
#
# if file_path is None:
# image_import.set_image_data(image, self.uri, None)
# return
# NOTE(flaper87): Don't assume the image was stored in the
# work_dir. Think in the case this path was provided by another task.
# Also, lets try to neither assume things nor create "logic"
# dependencies between this task and `_ImportToFS`
#
# base_path = os.path.dirname(file_path.split("file://")[-1])
# NOTE(flaper87): Hopefully just scenarios #3 and #4. I say
# hopefully because nothing prevents the user to use the same
# FS store path as a work dir
#
# image_path = os.path.join(base_path, image_id)
#
# if (base_path == CONF.glance_store.filesystem_store_datadir or
# base_path in CONF.glance_store.filesystem_store_datadirs):
# os.rename(file_path, image_path)
#
# image_import.set_image_data(image, image_path, None)
# NOTE(jokke): The different options here are kind of pointless as we
# will need the file path anyways for our delete workflow for now.
# For future proofing keeping this as is.
image = self.image_repo.get(self.image_id)
image_import.set_image_data(image, file_path or self.uri, self.task_id,
backend=self.backend)
# NOTE(flaper87): We need to save the image again after the locations
# have been set in the image.
self.image_repo.save(image)
def set_image_data(self, uri, task_id, backend, set_active, callback=None):
"""Populate image with data on a specific backend.
This is used during an image import operation to populate the data
in a given store for the image. If this object wraps an admin-capable
image_repo, then this will be done with admin credentials on behalf
of a user already determined to be able to perform this operation
(such as a copy-image import of an existing image owned by another
user).
:param uri: Source URL for image data
:param task_id: The task responsible for this operation
:param backend: The backend store to target the data
:param set_active: Whether or not to set the image to 'active'
state after the operation completes
:param callback: A callback function with signature:
fn(action, chunk_bytes, total_bytes)
which should be called while processing the image
approximately every minute.
"""
if callback:
callback = functools.partial(callback, self)
return image_import.set_image_data(self._image,
uri,
task_id,
backend=backend,
set_active=set_active,
callback=callback)
def test_set_image_data_http(self, mock_image_iter):
uri = 'http://www.example.com'
image = mock.Mock()
mock_image_iter.return_value = test_utils.FakeHTTPResponse()
self.assertIsNone(image_import_script.set_image_data(image,
uri,
None))
def test_set_image_data_with_callback(self, mock_gidi, mock_sw):
data = [b'0' * 60, b'0' * 50, b'0' * 10, b'0' * 150]
result_data = []
mock_gidi.return_value = iter(data)
mock_sw.return_value.expired.side_effect = [False, True, False, False]
image = mock.MagicMock()
callback = mock.MagicMock()
def fake_set_data(data_iter, **kwargs):
for chunk in data_iter:
result_data.append(chunk)
image.set_data.side_effect = fake_set_data
image_import_script.set_image_data(image,
'http://fake',
None,
callback=callback)
mock_gidi.assert_called_once_with('http://fake')
self.assertEqual(data, result_data)
# Since we only fired the timer once, only two calls expected
# for the four reads we did, including the final obligatory one
callback.assert_has_calls([mock.call(110, 110), mock.call(160, 270)])
def execute(self, image_id, file_path=None):
"""Bringing the introspected image to back end store
:param image_id: Glance Image ID
:param file_path: path to the image file
"""
# NOTE(flaper87): There are a couple of interesting bits in the
# interaction between this task and the `_ImportToFS` one. I'll try
# to cover them in this comment.
#
# NOTE(flaper87):
# `_ImportToFS` downloads the image to a dedicated `work_dir` which
# needs to be configured in advance (please refer to the config option
# docs for more info). The motivation behind this is also explained in
# the `_ImportToFS.execute` method.
#
# Due to the fact that we have an `_ImportToFS` task which downloads
# the image data already, we need to be as smart as we can in this task
# to avoid downloading the data several times and reducing the copy or
# write times. There are several scenarios where the interaction
# between this task and `_ImportToFS` could be improved. All these
# scenarios assume the `_ImportToFS` task has been executed before
# and/or in a more abstract scenario, that `file_path` is being
# provided.
#
# Scenario 1: FS Store is Remote, introspection enabled,
# conversion disabled
#
# In this scenario, the user would benefit from having the scratch path
# being the same path as the fs store. Only one write would happen and
# an extra read will happen in order to introspect the image. Note that
# this read is just for the image headers and not the entire file.
#
# Scenario 2: FS Store is remote, introspection enabled,
# conversion enabled
#
# In this scenario, the user would benefit from having a *local* store
# into which the image can be converted. This will require downloading
# the image locally, converting it and then copying the converted image
# to the remote store.
#
# Scenario 3: FS Store is local, introspection enabled,
# conversion disabled
# Scenario 4: FS Store is local, introspection enabled,
# conversion enabled
#
# In both these scenarios the user shouldn't care if the FS
# store path and the work dir are the same, therefore probably
# benefit, about the scratch path and the FS store being the
# same from a performance perspective. Space wise, regardless
# of the scenario, the user will have to account for it in
# advance.
#
# Lets get to it and identify the different scenarios in the
# implementation
image = self.image_repo.get(image_id)
image.status = 'saving'
self.image_repo.save(image)
# NOTE(flaper87): Let's dance... and fall
#
# Unfortunatelly, because of the way our domain layers work and
# the checks done in the FS store, we can't simply rename the file
# and set the location. To do that, we'd have to duplicate the logic
# of every and each of the domain factories (quota, location, etc)
# and we'd also need to hack the FS store to prevent it from raising
# a "duplication path" error. I'd rather have this task copying the
# image bits one more time than duplicating all that logic.
#
# Since I don't think this should be the definitive solution, I'm
# leaving the code below as a reference for what should happen here
# once the FS store and domain code will be able to handle this case.
#
# if file_path is None:
# image_import.set_image_data(image, self.uri, None)
# return
# NOTE(flaper87): Don't assume the image was stored in the
# work_dir. Think in the case this path was provided by another task.
# Also, lets try to neither assume things nor create "logic"
# dependencies between this task and `_ImportToFS`
#
# base_path = os.path.dirname(file_path.split("file://")[-1])
# NOTE(flaper87): Hopefully just scenarios #3 and #4. I say
# hopefully because nothing prevents the user to use the same
# FS store path as a work dir
#
# image_path = os.path.join(base_path, image_id)
#
# if (base_path == CONF.glance_store.filesystem_store_datadir or
# base_path in CONF.glance_store.filesystem_store_datadirs):
# os.rename(file_path, image_path)
#
# image_import.set_image_data(image, image_path, None)
image_import.set_image_data(image, file_path or self.uri, self.task_id)
# NOTE(flaper87): We need to save the image again after the locations
# have been set in the image.
self.image_repo.save(image)
def execute(self, image_id, file_path=None):
"""Bringing the introspected image to back end store
:param image_id: Glance Image ID
:param file_path: path to the image file
"""
# NOTE(flaper87): There are a couple of interesting bits in the
# interaction between this task and the `_ImportToFS` one. I'll try
# to cover them in this comment.
#
# NOTE(flaper87):
# `_ImportToFS` downloads the image to a dedicated `work_dir` which
# needs to be configured in advance (please refer to the config option
# docs for more info). The motivation behind this is also explained in
# the `_ImportToFS.execute` method.
#
# Due to the fact that we have an `_ImportToFS` task which downloads
# the image data already, we need to be as smart as we can in this task
# to avoid downloading the data several times and reducing the copy or
# write times. There are several scenarios where the interaction
# between this task and `_ImportToFS` could be improved. All these
# scenarios assume the `_ImportToFS` task has been executed before
# and/or in a more abstract scenario, that `file_path` is being
# provided.
#
# Scenario 1: FS Store is Remote, introspection enabled,
# conversion disabled
#
# In this scenario, the user would benefit from having the scratch path
# being the same path as the fs store. Only one write would happen and
# an extra read will happen in order to introspect the image. Note that
# this read is just for the image headers and not the entire file.
#
# Scenario 2: FS Store is remote, introspection enabled,
# conversion enabled
#
# In this scenario, the user would benefit from having a *local* store
# into which the image can be converted. This will require downloading
# the image locally, converting it and then copying the converted image
# to the remote store.
#
# Scenario 3: FS Store is local, introspection enabled,
# conversion disabled
# Scenario 4: FS Store is local, introspection enabled,
# conversion enabled
#
# In both these scenarios the user shouldn't care if the FS
# store path and the work dir are the same, therefore probably
# benefit, about the scratch path and the FS store being the
# same from a performance perspective. Space wise, regardless
# of the scenario, the user will have to account for it in
# advance.
#
# Lets get to it and identify the different scenarios in the
# implementation
image = self.image_repo.get(image_id)
image.status = 'saving'
self.image_repo.save(image)
# NOTE(flaper87): Let's dance... and fall
#
# Unfortunatelly, because of the way our domain layers work and
# the checks done in the FS store, we can't simply rename the file
# and set the location. To do that, we'd have to duplicate the logic
# of every and each of the domain factories (quota, location, etc)
# and we'd also need to hack the FS store to prevent it from raising
# a "duplication path" error. I'd rather have this task copying the
# image bits one more time than duplicating all that logic.
#
# Since I don't think this should be the definitive solution, I'm
# leaving the code below as a reference for what should happen here
# once the FS store and domain code will be able to handle this case.
#
# if file_path is None:
# image_import.set_image_data(image, self.uri, None)
# return
# NOTE(flaper87): Don't assume the image was stored in the
# work_dir. Think in the case this path was provided by another task.
# Also, lets try to neither assume things nor create "logic"
# dependencies between this task and `_ImportToFS`
#
# base_path = os.path.dirname(file_path.split("file://")[-1])
# NOTE(flaper87): Hopefully just scenarios #3 and #4. I say
# hopefully because nothing prevents the user to use the same
# FS store path as a work dir
#
# image_path = os.path.join(base_path, image_id)
#
# if (base_path == CONF.glance_store.filesystem_store_datadir or
# base_path in CONF.glance_store.filesystem_store_datadirs):
# os.rename(file_path, image_path)
#
# image_import.set_image_data(image, image_path, None)
image_import.set_image_data(image, file_path or self.uri, self.task_id)
# NOTE(flaper87): We need to save the image again after the locations
# have been set in the image.
self.image_repo.save(image)
def test_set_image_data_http(self):
uri = 'http://www.example.com'
image = mock.Mock()
self.assertEqual(None,
image_import_script.set_image_data(image, uri, None))
def test_set_image_data_http(self):
uri = 'http://www.example.com'
image = mock.Mock()
self.assertEqual(None,
image_import_script.set_image_data(image, uri, None))
def execute(self, file_path=None):
"""Bringing the imported image to back end store
:param image_id: Glance Image ID
:param file_path: path to the image file
"""
# NOTE(flaper87): Let's dance... and fall
#
# Unfortunatelly, because of the way our domain layers work and
# the checks done in the FS store, we can't simply rename the file
# and set the location. To do that, we'd have to duplicate the logic
# of every and each of the domain factories (quota, location, etc)
# and we'd also need to hack the FS store to prevent it from raising
# a "duplication path" error. I'd rather have this task copying the
# image bits one more time than duplicating all that logic.
#
# Since I don't think this should be the definitive solution, I'm
# leaving the code below as a reference for what should happen here
# once the FS store and domain code will be able to handle this case.
#
# if file_path is None:
# image_import.set_image_data(image, self.uri, None)
# return
# NOTE(flaper87): Don't assume the image was stored in the
# work_dir. Think in the case this path was provided by another task.
# Also, lets try to neither assume things nor create "logic"
# dependencies between this task and `_ImportToFS`
#
# base_path = os.path.dirname(file_path.split("file://")[-1])
# NOTE(flaper87): Hopefully just scenarios #3 and #4. I say
# hopefully because nothing prevents the user to use the same
# FS store path as a work dir
#
# image_path = os.path.join(base_path, image_id)
#
# if (base_path == CONF.glance_store.filesystem_store_datadir or
# base_path in CONF.glance_store.filesystem_store_datadirs):
# os.rename(file_path, image_path)
#
# image_import.set_image_data(image, image_path, None)
# NOTE(jokke): The different options here are kind of pointless as we
# will need the file path anyways for our delete workflow for now.
# For future proofing keeping this as is.
image = self.image_repo.get(self.image_id)
if image.status == "deleted":
raise exception.ImportTaskError("Image has been deleted, aborting"
" import.")
try:
image_import.set_image_data(image,
file_path or self.uri,
self.task_id,
backend=self.backend,
set_active=self.set_active)
# NOTE(yebinama): set_image_data catches Exception and raises from
# them. Can't be more specific on exceptions catched.
except Exception:
if not self.allow_failure:
raise
msg = (_("%(task_id)s of %(task_type)s failed but since "
"allow_failure is set to true, continue.") % {
'task_id': self.task_id,
'task_type': self.task_type
})
LOG.warning(msg)
if self.backend is not None:
failed_import = image.extra_properties.get(
'os_glance_failed_import', '').split(',')
failed_import.append(self.backend)
image.extra_properties['os_glance_failed_import'] = ','.join(
failed_import)
if self.backend is not None:
importing = image.extra_properties.get(
'os_glance_importing_to_stores', '').split(',')
try:
importing.remove(self.backend)
image.extra_properties[
'os_glance_importing_to_stores'] = ','.join(importing)
except ValueError:
LOG.debug(
"Store %s not found in property "
"os_glance_importing_to_stores.", self.backend)
# NOTE(flaper87): We need to save the image again after
# the locations have been set in the image.
self.image_repo.save(image)
