def get_metadata_catalog(api_session):
gc = glance_client(api_session)
catalog = [
namespace.__original__
for namespace in gc.metadefs_namespace.list(filters={'resource_types': 'OS::Glance::Image'})
]
for namespace in catalog:
ns_properties = []
ns_objects = []
for ns_property in gc.metadefs_property.list(namespace=namespace['namespace']):
ns_properties.append(ns_property)
if ns_properties:
namespace.update(properties=ns_properties)
for ns_object in gc.metadefs_object.list(namespace=namespace['namespace']):
ns_objects.append(ns_object)
if ns_objects:
namespace.update(objects=ns_objects)
return catalog
def glance_api(self):
"""
:rtype: glanceclient.v2.client.Client
"""
assert self.api_session is not None, 'Unable to use glance_api without an api_session!'
return glance_client(api_session=self.api_session, region_name=self.db_image.region.id)
def upload_empty_qcow2_image(self,
image_id,
region_name,
disk_size_in_bytes=0):
"""
Upload an image data to Glance as admin.
Do not use this task in user views.
"""
del self # unused
# ensure we have an empty qcow2 file to upload
if not path.isdir(settings.FLEIO_TEMP_DIR):
os.mkdir(settings.FLEIO_TEMP_DIR)
qcow2_file_path = path.join(settings.FLEIO_TEMP_DIR,
'boot_from_iso_img.qcow2')
if not path.isfile(qcow2_file_path):
subprocess.call([
'qemu-img', 'create', '-f', 'qcow2', qcow2_file_path,
str(disk_size_in_bytes)
])
if not path.isfile(qcow2_file_path):
raise IOError('Failed to initialize temporary qcow2 file')
admin_api = IdentityAdminApi()
gc = glance_client(api_session=admin_api.session, region_name=region_name)
with open(qcow2_file_path, mode="rb") as temp_file:
gc.images.upload(image_id=image_id, image_data=temp_file)
def upload_zero_filled_image(self, image_id, region_name, length_in_bytes=0):
"""
Upload an image data to Glance as admin.
Do not use this task in user views.
"""
del self # unused
admin_api = IdentityAdminApi()
gc = glance_client(api_session=admin_api.session, region_name=region_name)
buffer = BytesIO(bytearray(length_in_bytes))
gc.images.upload(image_id=image_id, image_data=buffer)
def create(self, owner, name, min_disk, min_ram, container_format='bare', disk_format='qcow2',
visibility='private', protected=False, architecture=None, os_distro=None, os_version=None, region=None,
hypervisor_type=None, file=None, url=None, source=None, properties=None):
optional_data = {}
if architecture:
optional_data['architecture'] = architecture
if os_distro:
optional_data['os_distro'] = os_distro
if hypervisor_type:
optional_data['hypervisor_type'] = hypervisor_type
if os_version:
optional_data['os_version'] = os_version
if properties:
optional_data.update(properties)
if region is not None:
region_name = region.id
else:
region_name = None
gc = glance_client(api_session=self.api_session, region_name=region_name)
# NOTE(tomo): First create the image, upload needs to happen after
openstack_image = gc.images.create(name=name, owner=owner, container_format=container_format,
disk_format=disk_format, min_disk=min_disk, min_ram=min_ram,
visibility=visibility, protected=protected, **optional_data)
if source == 'file' and file:
# save the uploaded file
image_temp_dir = getattr(settings, 'OPENSTACK_IMAGE_TEMP_DIR', None)
with tempfile.NamedTemporaryFile(prefix=openstack_image.id,
dir=image_temp_dir,
delete=False) as temp_image_file:
for chunk in file.chunks():
if chunk:
temp_image_file.write(chunk)
temp_image_file.close()
import_image_from_file.delay(openstack_image.id,
region_name=region_name,
file_path=temp_image_file.name)
elif source == 'url':
import_image_from_url.delay(openstack_image.id, url=url, region_name=region_name)
return openstack_image
def import_image_from_url(self, image_id, url, region_name):
"""
Upload an image data to Glance as admin, by downloading it first from a URL.
Do not use this task in user views.
"""
del self # unused
admin_api = IdentityAdminApi()
gc = glance_client(api_session=admin_api.session, region_name=region_name)
with requests.get(url, stream=True) as response:
response.raise_for_status()
temp_image_dir = getattr(settings, 'OPENSTACK_IMAGE_TEMP_DIR', None)
with tempfile.TemporaryFile(dir=temp_image_dir,
prefix='glance_') as temp_image_file:
for chunk in response.iter_content(chunk_size=4096):
if chunk:
temp_image_file.write(chunk)
temp_image_file.seek(
0) # Go to the beginning to be able to read it
gc.images.upload(image_id=image_id, image_data=temp_image_file)
def import_image_from_file(self, image_id, region_name, file_path):
"""
Upload an image data to Glance as admin.
Do not use this task in user views.
"""
del self # unused
LOG.info('Uploading image {} in region {} from file {}'.format(
image_id,
region_name,
file_path,
))
admin_api = IdentityAdminApi()
gc = glance_client(api_session=admin_api.session, region_name=region_name)
with open(file_path, mode="rb") as temp_file:
try:
gc.images.upload(image_id=image_id, image_data=temp_file)
finally:
filename = str(temp_file.name)
os.remove(filename)
def delete_openstack_image(self,
image_id,
region,
project_id,
user_id: Optional[int] = None):
del self # unused
project = Project.with_admin_session(project_id, cache=AUTH_CACHE)
gc = glance_client(api_session=project.api_session, region_name=region)
try:
db_image = ModelImage.objects.filter(id=image_id).first()
if db_image.protected:
gc.images.update(image_id=image_id, protected=False)
gc.images.delete(image_id=image_id)
user = AppUser.objects.filter(id=user_id).first()
if user and db_image:
if user.is_staff:
staff_signals.staff_delete_image.send(
sender=__name__,
user=user,
user_id=user.id,
image_name=db_image.name,
image_id=db_image.id,
username=user.username,
)
else:
user_signals.user_delete_image.send(
sender=__name__,
user=user,
user_id=user.id,
image_name=db_image.name,
image_id=db_image.id,
username=user.username,
)
except glance_exceptions.NotFound:
ModelImage.objects.filter(id=image_id).delete()
def update(self, image, **fields):
gc = glance_client(self.api_session, region_name=image.region.id)
fields_copy = copy.deepcopy(fields)
properties = fields_copy.pop('properties', None)
remove_props = []
non_removable_properties = ('owner', 'name', 'min_disk', 'min_ram', 'container_format', 'disk_format',
'visibility', 'protected', 'region')
for prop_name, prop_value in iter(fields.items()):
if prop_name not in non_removable_properties:
if prop_value is None or prop_value == '':
fields_copy.pop(prop_name)
remove_props.append(prop_name)
props = {}
if properties:
if isinstance(properties, str):
try:
properties = json.loads(properties)
except json.decoder.JSONDecodeError:
properties = {}
properties.pop('os_distro', None)
properties.pop('os_version', None)
properties.pop('hypervisor_type', None)
properties.pop('architecture', None)
props = copy.deepcopy(properties)
for name, value in iter(properties.items()):
if value is None or value == '':
remove_props.append(name)
props.pop(name)
# can't set and remove a property with the same value (bug in client), so first we must
# unset the variable, then set the new key
self.unset_properties_if_same_value(gc, image, properties, remove_props)
return gc.images.update(image.id, remove_props=remove_props, **fields_copy, **props)
def sync_images(region_id=None):
imh = ImageSyncHandler()
member_handler = ImageMemberSyncHandler()
gc = glance_client(api_session=get_keystone_admin().session, region_name=region_id, version='2')
# Retrieve images using pagination
has_more = True
list_limit = 50
marker = None
try:
while has_more:
images = gc.images.list(limit=list_limit, marker=marker)
if not images:
break
image_count = 0
for image in images:
imh.create_or_update(image, region=region_id, timestamp=timestamp)
marker = image.id
image_count += 1
# Sync image members here because we can only get members per image
if image.get('visibility') == 'shared':
for member in gc.image_members.list(image_id=image.id):
member_handler.create_or_update(member, region=region_id, timestamp=timestamp)
has_more = image_count == list_limit
except EndpointNotFound as e:
LOG.error(str(e))
return
version = imh.get_version(timestamp)
delete_filter = {'{}__lt'.format(imh.version_field): version, 'region__id': region_id}
imh.model_class.objects.filter(**delete_filter).delete()
# Delete older image member records
member_version = member_handler.get_version(timestamp)
member_delete_filter = {
'{}__lt'.format(member_handler.version_field): member_version,
'image__region__id': region_id
}
member_handler.model_class.objects.filter(**member_delete_filter).delete()
def download(self, image):
gc = glance_client(self.api_session, region_name=image.region.id)
return gc.images.data(image_id=image.id)
def reactivate(self, image):
gc = glance_client(self.api_session, region_name=image.region.id)
return gc.images.reactivate(image_id=image.id)
def run_backup(time_to_execute):
# TODO(MARIUS): implement command line arguments (recognisable flags like --force/-f)
# TODO(MARIUS): don't backup schedules from db that have already been executed (they pass the filter condition now)
# TODO(MARIUS): check backup types and execute the backups based on that, for accuracy
"""
:param time_to_execute: The time at which the procedure will be executed.
Scenario 1 (auto-run):
- an external automatic process (cron for linux, windows task scheduler for windows) will be set to execute this
script, in which :param time_to_execute: will be the time that the external process was set to run
e.g (5:00 AM, daily / weekly / etc.)
Scenario 2 (manual-run):
- script is run manually, in which case :param time_to_execute: will be the time at which the script is executed
All backups that are scheduled BEFORE the :param time_to_execute: are going to be executed.
"""
# default_session = admin_session()
backups_to_execute = OpenStackBackupSchedule.objects.filter(
run_at__lt=time_to_execute)
total_backups = len(backups_to_execute)
backups_executed = 0
backups_failed = []
session_map = dict(
# stores sessions based on project_id
)
for backup in backups_to_execute:
instance_region = backup.instance.region
project_id = backup.instance.project_id
# use the session related to the instance project_id
new_session = session_map.get(project_id, None)
if not new_session:
new_session = custom_session(project_id=project_id)
session_map[
project_id] = new_session # store the session in case we need it later
gc = glance_client(api_session=new_session,
region_name=instance_region)
nc = nova_client(api_session=new_session, region_name=instance_region)
nc.servers.backup(server=backup.instance.id,
backup_name=backup.backup_name,
backup_type=backup.backup_type,
rotation=backup.rotation)
if len(list(gc.images.list(filters={'name': backup.backup_name}))) > 0:
backups_executed += 1
else:
backups_failed.append(backup.instance.id)
logger.error(e)
logger.info(
'Cannot perform backup on instance {}, instance currently under a task state.'
.format(backup.instance.id))
time_to_execute = time_to_execute.strftime('%Y-%m-%d %H:%M:%S')
if not backups_failed:
msg = _(
'{executed} out of {total} backups were created successfully before {time_to_execute}'
).format(executed=backups_executed,
total=total_backups,
time_to_execute=time_to_execute)
else:
msg = _('Could not backup the following virtual machines: {failed}\n'
).format(backups_failed)
msg += _(
'Backed up {executed} out of {total} virtual machines').format(
executed=backups_executed, total=total_backups)
OpenStackBackupLog.objects.create()
return msg