def __init__(self, input_disk_path, working_dir): """Initialize azure disk object.""" # First initialize the super class. super().__init__(input_disk_path, working_dir) self.uploaded_disk_url = None self.connection_string = get_config_value('AZURE_STORAGE_CONNECTION_STRING') self.container_name = get_config_value('AZURE_STORAGE_CONTAINER_NAME') self.blob = None self.progress_cb_lu = 0 self.metadata = CloudImageMetadata()
def initialize_image_metadata(self, artifacts_dir): """Initialize image metadata""" self.metadata = CloudImageMetadata() self.metadata.load_artifact_files(artifacts_dir) # Set common metadata values self.metadata.set(self.__class__.__name__, 'input', self.image_disk_path) self.metadata.set(self.__class__.__name__, 'image_name', self.image_name) if os.getenv('CI') is None: self.metadata.set(self.__class__.__name__, 'build_type', 'local') else: self.metadata.set(self.__class__.__name__, 'build_type', 'pipeline') # License model is currently hardwired self.metadata.set(self.__class__.__name__, 'license_model', 'byol')
def __init__(self, input_disk_path, working_dir): """Initialize azure disk object.""" # First initialize the super class. super().__init__(input_disk_path, working_dir) self.connection_string = get_config_value('AZURE_STORAGE_CONNECTION_STRING') self.container_name = get_config_value('AZURE_STORAGE_CONTAINER_NAME') try: self.svc = PageBlobService(connection_string=self.connection_string) except ValueError: LOGGER.error("Could not create a PageBlobService with connection_string=%s", self.connection_string) raise RuntimeError("Runtime Error during Instantiating Azure Blob Service") self.progress_cb_lu = 0 self.metadata = CloudImageMetadata()
def __init__(self, working_dir, input_disk_path): super().__init__(working_dir, input_disk_path) self.session = Session( aws_access_key_id=get_config_value('AWS_ACCESS_KEY_ID'), aws_secret_access_key=get_config_value('AWS_SECRET_ACCESS_KEY'), region_name=get_config_value('AWS_REGION')) self.disk = AWSDisk(input_disk_path, working_dir, self.session) # Create ec2 client object for performing low-level image actions. self.ec2_client = self.session.client('ec2') # Record REGION in the metadata. self.metadata = CloudImageMetadata() self.metadata.set(self.__class__.__name__, 'location', self.session.region_name) self.snapshot = None self.image_id = None
def __init__(self, working_dir, input_disk_path): super().__init__(working_dir, input_disk_path) self.disk = GoogleDisk(input_disk_path) # Retrieve credentials dictionary creds_dict = get_dict_from_config_json( "GOOGLE_APPLICATION_CREDENTIALS") # Obtain project ID from the credentials dictionary self.gce_project_id = ensure_value_from_dict(creds_dict, "project_id") LOGGER.info("Using project_id: '%s'", self.gce_project_id) # Record project ID in metadata self.metadata = CloudImageMetadata() self.metadata.set(self.__class__.__name__, 'gce_project', self.gce_project_id) # Create a service object from the credentials dictionary self.gce_credentials = service_account.Credentials.from_service_account_info( creds_dict) self.gce_service = discovery.build('compute', 'v1', credentials=self.gce_credentials)
def __init__(self, ec2_client, s3_bucket, s3_disk): self.s3_bucket = s3_bucket self.s3_disk = s3_disk # ec2_client object to perform various EC2 operations. self.ec2_client = ec2_client LOGGER.trace("self.s3_disk = '%s'", self.s3_disk) LOGGER.trace("self.s3_bucket = '%s'", self.s3_bucket) # Snapshot created from the uploaded 's3_disk'. self.snapshot_id = None # Import-task-id for the disk import operation. self.import_task_id = None # Snapshot metadata. self.metadata = CloudImageMetadata()
class GoogleImage(BaseImage): """ Google class for all platform specific derivations """ def __init__(self, working_dir, input_disk_path): super().__init__(working_dir, input_disk_path) self.disk = GoogleDisk(input_disk_path) # Retrieve credentials dictionary creds_dict = get_dict_from_config_json( "GOOGLE_APPLICATION_CREDENTIALS") # Obtain project ID from the credentials dictionary self.gce_project_id = ensure_value_from_dict(creds_dict, "project_id") LOGGER.info("Using project_id: '%s'", self.gce_project_id) # Record project ID in metadata self.metadata = CloudImageMetadata() self.metadata.set(self.__class__.__name__, 'gce_project', self.gce_project_id) # Create a service object from the credentials dictionary self.gce_credentials = service_account.Credentials.from_service_account_info( creds_dict) self.gce_service = discovery.build('compute', 'v1', credentials=self.gce_credentials) def clean_up(self): """Clean-up cloud objects created by this class and its members.""" LOGGER.info("Cleaning-up GoogleImage artifacts.") if self.disk is not None: self.disk.clean_up() LOGGER.info("Completed GoogleImage clean-up.") def extract_disk(self): """Extract disk for upload""" BaseImage.extract_disk(self) # returns True if image exists, False otherwise def image_exists(self, image_name): """Check if image already exists in gce""" try: # pylint: disable=no-member request = self.gce_service.images().get( project=self.gce_project_id, image=image_name) result = request.execute() if not result: result = False else: result = True except HttpError as exp: if exp.resp.status == 404: result = False else: raise exp return result def is_image_deleted(self, image_name): """Waits for the image to be deleted.""" retrier = Retrier(lambda s: not self.image_exists(s), image_name) retrier.tries = int( get_config_value('GCE_IMAGE_DELETE_COMPLETED_RETRY_COUNT')) retrier.delay = int( get_config_value('GCE_IMAGE_DELETE_COMPLETED_RETRY_DELAY')) LOGGER.info('Waiting for image [%s] to be deleted.', image_name) try: if retrier.execute(): LOGGER.info("Image [%s] was deleted.", image_name) return True LOGGER.warning( "Image [%s] was still not deleted after checking [%d] times!", image_name, retrier.tries) return False except HttpError as exp: LOGGER.exception(exp) return False # delete the image, then wait for the deletion to complete # returns True if deletion was successful or image does not exist, False otherwise def delete_image(self, image_name): """Delete image from GCE""" image_deleted = False try: # pylint: disable=no-member request = self.gce_service.images().delete(project=self.gce_project_id, \ image=image_name) request.execute() except HttpError as exp: if exp.resp.status == 404: LOGGER.info("Image doesn't exist") image_deleted = True else: LOGGER.exception(exp) raise exp return image_deleted return self.is_image_deleted(image_name) def insert_image(self, image_name): """Create image in GCE and then check for status = READY""" bucket_name = get_config_value('GCE_BUCKET') image_body = { "name": image_name, "rawDisk": { # In the following line the bucket name along with blob name is required "source": "https://storage.googleapis.com/{}/{}".format( bucket_name, self.disk.uploaded_disk_name) } } try: # pylint: disable=no-member request = self.gce_service.images().insert( project=self.gce_project_id, body=image_body) result = request.execute() except HttpError as exp: LOGGER.exception(exp) raise exp if not result: return False LOGGER.debug("Image creation response: '%s'", result) return self.is_image_ready(image_name) def is_image_ready(self, image_name): """Checks if the given image is ready.""" def _is_image_ready(): """Checks if an image with image_name exists and status is READY""" # pylint: disable=no-member request = self.gce_service.images().get( project=self.gce_project_id, image=image_name) result = request.execute() if not result or result['status'] == 'FAILED': raise RuntimeError( "Creation of image [{}] failed!".format(image_name)) return result['status'] == 'READY' retrier = Retrier(_is_image_ready) retrier.tries = int( get_config_value('GCE_IMAGE_CREATE_COMPLETED_RETRY_COUNT')) retrier.delay = int( get_config_value('GCE_IMAGE_CREATE_COMPLETED_RETRY_DELAY')) LOGGER.info("Waiting for image [%s] to be ready.", image_name) try: if retrier.execute(): LOGGER.info("Image [%s] is ready.", image_name) self.metadata.set(self.__class__.__name__, 'image_id', image_name) return True LOGGER.warning( "Image [%s] was still not ready after checking [%d] times!", image_name, retrier.tries) return False except HttpError as exp: LOGGER.exception(exp) return False except RuntimeError as runtime_exception: LOGGER.exception(runtime_exception) return False def tag_image(self, image_name): """Associate image tags with image""" LOGGER.info('Set image labels.') # Get current labels fingerprint. To avoid/detect conflicts, you must # provide the current label fingerprint (reference) when you request to # set image labels. This fingerprint value is updated whenever labels # are updated and the set labels request will fail if the labels were # updated out of band. try: # pylint: disable=no-member request = self.gce_service.images().get( project=self.gce_project_id, image=image_name) result = request.execute() label_fingerprint = result['labelFingerprint'] except HttpError as exp: LOGGER.error("Exception setting image labels:") LOGGER.exception(exp) return False if not result: return False if label_fingerprint is None or label_fingerprint == '': LOGGER.info('Label fingerprint was empty.') return False cloud_image_tags = CloudImageTags(self.metadata) cloud_image_tags.transform_values(to_lower=True, disallowed_regex='[^a-z0-9-]') image_labels = cloud_image_tags.get() set_labels_body = { "labels": image_labels, "labelFingerprint": label_fingerprint } try: # pylint: disable=no-member request = self.gce_service.images().setLabels( project=self.gce_project_id, resource=image_name, body=set_labels_body) result = request.execute() except HttpError as exp: LOGGER.error("Exception setting image labels:") LOGGER.exception(exp) return False if not result: return False LOGGER.debug("Image set labels response: %s", result) return True def create_image(self, image_name): LOGGER.info("Checking if the image '%s' already exists.", image_name) # Check if an image with image_name already exists. If so, delete the image result = self.image_exists(image_name) if not result: LOGGER.info("The image '%s' does not exist.", image_name) else: LOGGER.info("The image '%s' exists.", image_name) result = self.delete_image(image_name) if not result: LOGGER.error("Could not delete the image '%s', exiting.", image_name) raise SystemExit(-1) LOGGER.info("Attempting to create an image '%s'.", image_name) result = self.insert_image(image_name) if not result: LOGGER.error("The image '%s' was not created successfully.", image_name) raise SystemExit(-1) result = self.tag_image(image_name) if not result: LOGGER.error("The image '%s' was not tagged successfully.", image_name) raise SystemExit(-1) LOGGER.info("Image '%s' creation succeeded.", image_name)
class AWSImage(BaseImage): """Class for handling AWS image related actions""" AWS_IMAGE_ROOT_VOLUME = '/dev/xvda' def __init__(self, working_dir, input_disk_path): super().__init__(working_dir, input_disk_path) self.session = Session( aws_access_key_id=get_config_value('AWS_ACCESS_KEY_ID'), aws_secret_access_key=get_config_value('AWS_SECRET_ACCESS_KEY'), aws_session_token=get_config_value('AWS_SESSION_TOKEN'), region_name=get_config_value('AWS_REGION') ) self.disk = AWSDisk(input_disk_path, working_dir, self.session) # Create ec2 client object for performing low-level image actions. self.ec2_client = self.session.client('ec2') # Record REGION in the metadata. self.metadata = CloudImageMetadata() self.metadata.set(self.__class__.__name__, 'location', self.session.region_name) self.snapshot = None self.image_id = None def clean_up(self): """Clean-up cloud objects created by this class and its members.""" LOGGER.info("Cleaning-up AWSImage artifacts.") # Clean-up the snapshot only if image generation didn't succeed. In case of a # successful image generation, the snapshot is associated with the image and # stays in-use. Trying to delete it in that case would always fail. if self.image_id is None and self.snapshot is not None: self.snapshot.clean_up() if self.disk is not None: self.disk.clean_up() LOGGER.info("Completed AWSImage clean-up.") def create_image(self, image_name): """Create image implementation for AWS""" # image name must be unique self.delete_old_image(image_name) #start image creation LOGGER.info('Started creation of image %s at %s', image_name, datetime.datetime.now().strftime('%H:%M:%S')) start_time = time() try: response = self.ec2_client.register_image( Architecture="x86_64", BlockDeviceMappings=[ { "DeviceName": AWSImage.AWS_IMAGE_ROOT_VOLUME, "Ebs": { "DeleteOnTermination": True, "SnapshotId": self.snapshot.snapshot_id, "VolumeType": "gp2" } } ], EnaSupport=True, Description=image_name, Name=image_name, RootDeviceName=AWSImage.AWS_IMAGE_ROOT_VOLUME, SriovNetSupport="simple", VirtualizationType="hvm" ) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('register_image failed for image\'{}\'!'.format(image_name)) \ from botocore_exception # get image id try: LOGGER.trace("register_image() response: %s", response) self.image_id = response['ImageId'] except KeyError as key_error: LOGGER.exception(key_error) raise RuntimeError('could not find \'ImageId\' key for image {} '.format(image_name) + 'in create_image response: {}'.format(response)) from key_error LOGGER.info('Image id: %s', self.image_id) # save image id in artifacts dir json file save_image_id(self.image_id) # wait till the end of the image creation self.wait_for_image_availability() LOGGER.info('Creation of %s image took %d seconds', self.image_id, time() - start_time) LOGGER.info('Tagging %s as the image_id.', self.image_id) self.metadata.set(self.__class__.__name__, 'image_id', self.image_id) # add tags to the image self.create_tags() def delete_old_image(self, image_name): """ Check if an image with the same name already exists and delete it. This is unlikely to happen unless the image name is specified in the configuration.""" response = self.find_image(image_name) num_images = len(response['Images']) if num_images not in (0, 1, 2): raise RuntimeError('Number of images named {} '.format(image_name) + 'expected to be 0 or 1 (maybe 2, due to AWS replicability issues),' + ' but found {}. '.format(num_images) + '(Should have received InvalidAMIName.Duplicate error during ' + 'the previous image creation). Please delete them manually.') if num_images in (1, 2): try: first_image_id = response['Images'][0]['ImageId'] if num_images == 2: second_image_id = response['Images'][1]['ImageId'] except KeyError as key_error: LOGGER.exception(key_error) raise RuntimeError( 'could not find ImageId key for image {} '.format(image_name) + 'in describe_images response: {}'.format(response)) from key_error LOGGER.info('There is an old image %s named %s, deleting it.', first_image_id, image_name) self.delete_image(first_image_id) if num_images == 2: LOGGER.info('There is an old image %s named %s, deleting it.', second_image_id, image_name) self.delete_image(second_image_id) def find_image(self, image_name): """ Find image by name. Return response""" try: response = self.ec2_client.describe_images( Filters=[{'Name': 'name', 'Values':[image_name]}]) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('describe_images failed for image \'{}\' !'.format(image_name)) \ from botocore_exception LOGGER.trace('describe_images response for image %s: %s', image_name, response) return response def delete_image(self, image_id): """ Delete image by image id. Return response""" try: self.ec2_client.deregister_image(ImageId=image_id) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('deregister_image failed for image \'{}\' !'.format(image_id)) \ from botocore_exception def wait_for_image_availability(self): """ Wait for image to be created and available """ def _wait_for_image_availability(): """Awaits the describe_images() to successfully acknowledge availability of the given image.""" try: response = self.ec2_client.describe_images(ImageIds=[self.image_id]) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('EC2.Client.describe_images() failed for {} !'. format(self.image_id)) from botocore_exception if not response: raise RuntimeError('EC2.Client.describe_images() returned none response!') \ from botocore_exception try: if response['Images'][0]['State'] == 'available': return True return False except (KeyError, IndexError) as image_describe_exception: LOGGER.exception(image_describe_exception) raise RuntimeError('EC2.Client.describe_images() did not have ' + '[\'Images\'][0][\'State\'] in its response: response \'{}\''. format(response)) from image_describe_exception retrier = Retrier(_wait_for_image_availability) retrier.tries = int(get_config_value('AWS_CREATE_IMAGE_RETRY_COUNT')) retrier.delay = int(get_config_value('AWS_CREATE_IMAGE_RETRY_DELAY')) LOGGER.info('Waiting for the image %s to become available.', self.image_id) if retrier.execute(): LOGGER.info('Image [%s] is created in AWS.', self.image_id) else: raise RuntimeError('Exhausted all \'{}\' retries for image {} to become available.'. format(self.image_id, retrier.tries)) def get_image_tag_metadata(self): """Returns associated image metadata tags through the member variable metadata.""" metadata_tags = CloudImageTags(self.metadata) return metadata_tags.get() def create_tags(self): """ Create tags for image. Tags are fetched from metadata. """ image_tags = self.get_image_tag_metadata() tags_to_add = [] for tag in image_tags: tags_to_add.append({'Key': tag, 'Value': image_tags[tag]}) try: response = self.ec2_client.create_tags(Resources=[self.image_id], Tags=tags_to_add) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('create_tags failed for image\'{}\'!\n'.format(self.image_id)) \ from botocore_exception LOGGER.trace('create_tags response for image %s: %s', self.image_id, response) def prep_disk(self): """Performs the leg work to convert the S3 Disk represented by self.disk into a snapshot from which an AWSImage can be created.""" LOGGER.info("Prepare the uploaded s3 disk for image generation.") # Convert the s3Disk into an AWS Snapshot. self.snapshot = AWSSnapshot(self.ec2_client, self.disk.bucket_name, self.disk.uploaded_disk_name) self.snapshot.create_snapshot() LOGGER.info("AWS Disk preparation is complete for image creation.") def share_image(self): """Reads a list of AWS accounts and shares the AMI with each of those accounts.""" share_account_ids = get_list_from_config_yaml('AWS_IMAGE_SHARE_ACCOUNT_IDS') if share_account_ids: LOGGER.info("Share the AMI with multiple AWS accounts.") for dest_account_id in share_account_ids: try: LOGGER.info('Sharing image with account-id: %s', dest_account_id) # Share the image with the destination account response = self.ec2_client.modify_image_attribute( ImageId=self.image_id, Attribute='launchPermission', OperationType='add', UserIds=[str(dest_account_id)] ) LOGGER.trace("image.modify_attribute response => %s", response) except ClientError as client_error: LOGGER.exception(client_error) # Log the error around malformed Account-id and move on. if client_error.response['Error']['Code'] == 'InvalidAMIAttributeItemValue': LOGGER.error('Malformed account-id: %s', dest_account_id) else: # Any other type of error can be irrecoverable and might # point to a deeper malaise. raise RuntimeError('aws IMAGE was not shared with other accounts') \ from client_error # Acknowledge all the account-ids that the image was shared with. self.is_share_image_succeeded(share_account_ids) else: LOGGER.info("No account IDs found for sharing AMI") def is_share_image_succeeded(self, share_account_ids): """Helper utility for share_image() that goes through the list of share_account_ids and confirms that the image was shared with all accounts. The function logs any error during its execution without propagating it up.""" try: LOGGER.info("Checking which accounts were added for sharing this AMI") image_launch_perms = self.ec2_client.describe_image_attribute( ImageId=self.image_id, Attribute='launchPermission', DryRun=False ) LOGGER.trace("image.describe_attribute() response => %s", image_launch_perms) except ClientError as client_error: # Simply log the exception without propagating it. LOGGER.exception(client_error) return False # Create a list of account IDs that has launch permission launch_permission_accounts = [] for each in image_launch_perms['LaunchPermissions']: launch_permission_accounts.append(each['UserId']) counter = 0 # Check which accounts were added for sharing this AMI for account_id in share_account_ids: if str(account_id) in launch_permission_accounts: LOGGER.info("The AMI was successfully shared with account: %s", account_id) counter += 1 else: LOGGER.warning("The AMI was not shared with account: %s", account_id) # Confirm that the number of accounts in share_account_ids and image's # 'LaunchPermissions' are matching. return counter == len(share_account_ids)
def create_image(self, image_name): """ Create image implementation for Alibaba """ images_json = self.client.describe_images(None, image_name) if int(images_json['TotalCount']) == 0: LOGGER.debug('No old images named \'%s\' were found', image_name) else: # image names are unique, delete only one image image_id = images_json['Images']['Image'][0]['ImageId'] LOGGER.info('Image \'%s\' already exists, its id is \'%s\', deleting it', image_name, image_id) self.client.delete_image(image_id) # start image creation LOGGER.info('Started creation of image \'%s\' at %s', image_name, datetime.datetime.now().strftime('%H:%M:%S')) start_time = time() imported_image = self.client.import_image(get_config_value('ALIBABA_BUCKET'), self.disk.uploaded_disk_name, image_name) if 'Code' in imported_image.keys(): if imported_image['Code'] == 'InvalidOSSObject.NotFound': raise RuntimeError('ImportImageRequest could not find uloaded disk \'' + image_name + '\'') if imported_image['Code'] == 'InvalidImageName.Duplicated': raise RuntimeError('Image \'' + image_name + '\' still exists, ' + 'should have been removed by this point') if imported_image['Code'] == 'ImageIsImporting': raise RuntimeError('Another image named \'' + image_name + '\' is in the ' + 'process of importing, probably from the previous run. ' + 'Delete it first.') if 'ImageId' not in imported_image.keys() or 'TaskId' not in imported_image.keys(): LOGGER.info('Alibaba response to ImportImageRequest:') LOGGER.info(json.dumps(imported_image, sort_keys=True, indent=4, separators=(',', ': '))) raise RuntimeError('ImageId and/or TaskId were not found in the response ' + 'cannot initiate image import') self.image_id = imported_image['ImageId'] task_id = imported_image['TaskId'] LOGGER.info('Started image import with image id \'%s\' and task id \'%s\'', self.image_id, task_id) task_status_count = int(get_config_value('ALIBABA_IMAGE_IMPORT_MONITOR_RETRY_COUNT')) task_status_delay = int(get_config_value('ALIBABA_IMAGE_IMPORT_MONITOR_RETRY_DELAY')) if self.monitor_task(task_id, task_status_count, task_status_delay): LOGGER.info('Image \'%s\' imported after %d seconds', self.image_id, time() - start_time) else: canceled_task_msg = 'Image import failed or took too long, ' + \ 'canceling task \'{}\' and '.format(task_id) + \ 'deleting image \'{}\''.format(self.image_id) LOGGER.info(canceled_task_msg) self.client.cancel_task(task_id) self.client.delete_image(self.image_id) raise RuntimeError('Failed to import image \'{}\' after monitoring it for {} retries'. format(self.image_id, task_status_count)) # Add image_id and location (region) to the metadata used for image registration metadata = CloudImageMetadata() metadata.set(self.__class__.__name__, 'image_id', self.image_id) metadata.set(self.__class__.__name__, 'location', get_config_value('ALIBABA_REGION')) # Add tags to image LOGGER.info('Add tags to image \'%s\'', self.image_id) self.client.add_tags(self.image_id, 'image', CloudImageTags(metadata).get()) # Add tags to associated snapshot images_json = self.client.describe_images(self.image_id, None) if not 'Images' in images_json.keys(): LOGGER.error('No image data found for image \'%s\'', self.image_id) LOGGER.error('Unable to tag snapshot.') else: snapshot_id = images_json['Images']['Image'][0] \ ['DiskDeviceMappings']['DiskDeviceMapping'][0]['SnapshotId'] LOGGER.info('Add tags to snapshot \'%s\'', snapshot_id) self.client.add_tags(snapshot_id, 'snapshot', CloudImageTags(metadata).get())
class AzureDisk(BaseDisk): """ Manage Azure disk """ # pylint: disable=too-many-instance-attributes def __init__(self, input_disk_path, working_dir): """Initialize azure disk object.""" # First initialize the super class. super().__init__(input_disk_path, working_dir) self.uploaded_disk_url = None self.connection_string = get_config_value( 'AZURE_STORAGE_CONNECTION_STRING') self.container_name = get_config_value('AZURE_STORAGE_CONTAINER_NAME') try: self.svc = PageBlobService( connection_string=self.connection_string) except ValueError: LOGGER.error( "Could not create a PageBlobService with connection_string=%s", self.connection_string) raise RuntimeError( "Runtime Error during Instantiating Azure Blob Service") self.progress_cb_lu = 0 self.metadata = CloudImageMetadata() def clean_up(self): """Clean-up the uploaded disk after image generation.""" def set_uploaded_disk_name(self, disk_name): """Set the uploaded disk name""" # As Azure disk takes its name from the image-name (unlike other clouds where # the disk-name are auto-generated during disk extraction), append disk extension # to the uploaded disk name. self.uploaded_disk_name = disk_name + '.vhd' LOGGER.info("The uploaded disk name is '%s'.", self.uploaded_disk_name) def extract(self): """Extract the vhd disk out of tar.gz.""" self.disk_to_upload = BaseDisk.decompress(self.input_disk_path, '.vhd', self.working_dir) LOGGER.info("Azure disk_to_upload = '%s'", self.disk_to_upload) def _get_tags(self): tags = CloudImageTags(self.metadata) tags.title_case_keys() return tags.get() def _progress_cb(self, byte_up, byte_total): sec = int(time()) # No update within 10 second interval if sec - self.progress_cb_lu > 10: self.progress_cb_lu = sec byte_up //= (1 << 20) byte_total //= (1 << 20) LOGGER.info('Uploaded %d MB of total %d MB', byte_up, byte_total) def upload(self): """ Upload a F5 BIG-IP VE image to provided container """ def _upload_impl(): """ Azure blob upload implementation """ cnum = int( get_config_value('AZURE_BLOB_UPLOAD_CONCURRENT_THREAD_COUNT')) timeout = int(get_config_value('AZURE_BLOB_UPLOAD_TIMEOUT')) try: self.svc.create_blob_from_path(self.container_name, self.uploaded_disk_name, \ self.disk_to_upload, max_connections=cnum, \ metadata=self._get_tags(), progress_callback=self._progress_cb, \ timeout=timeout) uploaded_blob = self.svc.get_blob_properties(self.container_name, \ self.uploaded_disk_name) uploaded_blob_size = uploaded_blob.properties.content_length local_blob_size = getsize(self.disk_to_upload) LOGGER.info("uploaded blob size: %s and local blob_size: %s", \ str(uploaded_blob_size), str(local_blob_size)) if uploaded_blob_size != local_blob_size: return False except AzureMissingResourceHttpError: LOGGER.error("Exception during uploading %s", self.disk_to_upload) return False except AzureException: LOGGER.error("Exception during uploading %s", self.disk_to_upload) return False self.uploaded_disk_url = self.svc.make_blob_url( self.container_name, self.uploaded_disk_name) # save uploaded disk in artifacts dir json file vhd_url_json = {"vhd_url": self.uploaded_disk_url} artifacts_dir = get_config_value("ARTIFACTS_DIR") with open(artifacts_dir + "/vhd_url.json", "w") as vhd_url_json_file: json.dump(vhd_url_json, vhd_url_json_file) # insert file with vhd url self.metadata.set(self.__class__.__name__, 'vhd_url', self.uploaded_disk_url) self.metadata.set(self.__class__.__name__, 'image_id', self.uploaded_disk_name) LOGGER.info('Uploaded disk url is: %s', self.uploaded_disk_url) return True retrier = Retrier(_upload_impl) retrier.tries = int( get_config_value('AZURE_BLOB_UPLOAD_COMPLETED_RETRY_COUNT')) retrier.delay = int( get_config_value('AZURE_BLOB_UPLOAD_COMPLETED_RETRY_DELAY')) LOGGER.info("Waiting for blob %s to be uploaded.", self.disk_to_upload) if retrier.execute(): LOGGER.info("blob [%s] is ready.", self.disk_to_upload) return True LOGGER.error( "blob [%s] was still not ready after checking [%d] times!", self.disk_to_upload, retrier.tries) raise RuntimeError("Runtime Error Occured during Azure Disk Upload")
class ImageController(): # pylint: disable=too-many-instance-attributes """Controller to prepare cloud image""" def __init__(self, artifacts_dir, cloud_type, image_disk_path, should_clean=True): self.start_time = time.time() self.artifacts_dir = artifacts_dir self.cloud_type = cloud_type self.image_name = None self.image_disk_path = image_disk_path self.metadata = None self.status = 'failure' self.transformed_image_name = None self.working_dir = None self.should_clean = should_clean self.cloud_image = None if not os.path.isdir(artifacts_dir): raise ValueError( "Missing or invalid artifacts directory '{}'.".format( artifacts_dir)) if not os.path.isfile(image_disk_path): raise ValueError( "Missing image disk '{}'.".format(image_disk_path)) # Create a working directory under the artifacts dir to temporarily store # various build constructs and files. self.create_working_dir(artifacts_dir) try: # Factory (could be a separate object) # pylint: disable=import-outside-toplevel if cloud_type == 'alibaba': from image.alibaba_image import AlibabaImage self.cloud_image = AlibabaImage(self.working_dir, self.image_disk_path) elif cloud_type == 'aws': from image.aws_image import AWSImage self.cloud_image = AWSImage(self.working_dir, self.image_disk_path) elif cloud_type == 'azure': from image.azure_image import AzureImage self.cloud_image = AzureImage(self.working_dir, self.image_disk_path) elif cloud_type == 'gce': from image.google_image import GoogleImage self.cloud_image = GoogleImage(self.working_dir, self.image_disk_path) else: raise ValueError( 'Unexpected cloud type: {}'.format(cloud_type)) # pylint: enable=import-outside-toplevel self.cloud_image_name = self.image_name_factory(cloud_type) except BaseException as base_exception: LOGGER.exception(base_exception) raise base_exception def clean_up(self): """Cleans-up the cloud and local artifacts created by this object.""" try: if self.should_clean is True: if self.cloud_image is not None: LOGGER.info("Cleaning up image controller constructs.") self.cloud_image.clean_up() self.cloud_image = None if self.working_dir is not None and os.path.isdir( self.working_dir): LOGGER.debug("Removing working dir '%s'.", self.working_dir) shutil.rmtree(self.working_dir) self.working_dir = None else: LOGGER.debug("Skipping removal of working dir '%s'.", self.working_dir) except OSError as os_exception: # Simply log the exception without propagating. LOGGER.error(os_exception) @staticmethod def image_name_factory(cloud_type): """Factory pattern for ImageName""" # pylint: disable=import-outside-toplevel if cloud_type == 'alibaba': from image.alibaba_image_name import AlibabaImageName return AlibabaImageName() if cloud_type == 'aws': from image.aws_image_name import AWSImageName return AWSImageName() if cloud_type == 'azure': from image.azure_image_name import AzureImageName return AzureImageName() if cloud_type == 'gce': from image.google_image_name import GoogleImageName return GoogleImageName() raise ValueError('Unexpected cloud type: {}'.format(cloud_type)) # pylint: enable=import-outside-toplevel @staticmethod def check_valid_name(cloud_type, user_image_name): """Check if user-supplied image name is valid""" cloud_image_name = ImageController.image_name_factory(cloud_type) cloud_image_name.check_valid_name(user_image_name) def set_image_name(self, seed_image_name='', user_image_name=''): """Set/Transform image name""" if user_image_name != '': user_image_name = user_image_name.strip() self.cloud_image_name.check_valid_name(user_image_name) self.image_name = user_image_name else: if seed_image_name == '': raise ValueError( 'seed_image_name or user_image_name is required') self.image_name = \ self.cloud_image_name.apply_transform(seed_image_name.strip())[0] def initialize_image_metadata(self, artifacts_dir, pipeline_build=False): """Initialize image metadata""" self.metadata = CloudImageMetadata() self.metadata.load_artifact_files(artifacts_dir) # Set common metadata values self.metadata.set(self.__class__.__name__, 'input', self.image_disk_path) self.metadata.set(self.__class__.__name__, 'image_name', self.image_name) if pipeline_build is True: self.metadata.set(self.__class__.__name__, 'build_type', 'pipeline') else: self.metadata.set(self.__class__.__name__, 'build_type', 'local') # License model is currently hardwired self.metadata.set(self.__class__.__name__, 'license_model', 'byol') def prepare(self, seed_image_name='', user_image_name=''): """Main controller""" try: self.set_image_name(seed_image_name, user_image_name) LOGGER.info("Starting prepare cloud image '%s'.", self.image_name) self.cloud_image.set_uploaded_disk_name(self.image_name) pipeline_build = os.getenv('CI') is not None self.initialize_image_metadata(self.artifacts_dir, pipeline_build) self.cloud_image.extract_disk() self.cloud_image.upload_disk() self.cloud_image.prep_disk() self.metadata.set(self.__class__.__name__, 'build_operation', 'create') self.cloud_image.create_image(self.image_name) build_time = time.time() - self.start_time self.metadata.set(self.__class__.__name__, 'build_time', str(timedelta(seconds=build_time))) self.status = 'success' self.metadata.set(self.__class__.__name__, 'status', self.status) self.cloud_image.share_image() self.create_metadata() self.register_image() self.create_report() LOGGER.info("Finished prepare cloud image '%s'.", self.image_name) except BaseException as base_exception: LOGGER.exception(base_exception) raise base_exception def create_working_dir(self, artifacts_dir): """Create temporary directory""" self.working_dir = tempfile.mkdtemp('', "image_" + self.cloud_type + "_", artifacts_dir) LOGGER.debug("Working directory = '%s'.", self.working_dir) def register_image(self): """Register image with CIR""" CloudImageRegister(self.metadata).register_image() def create_metadata(self): """Create metadata file""" # initialize input/output input_metadata = self.metadata.get() output_metadata = {'status': self.status} # Place metadata from self and cloud image class in output metadata build_sources = [ self.__class__.__name__, self.cloud_image.__class__.__name__ ] for build_source in build_sources: if build_source in input_metadata: output_metadata.update(input_metadata[build_source]) # Write metadata file metadata_file = '{}/prepare_cloud_image.json'.format( self.artifacts_dir) with open(metadata_file, 'w') as metadata_fp: json.dump(output_metadata, metadata_fp, indent=4, sort_keys=True) def create_report(self): """Create report of created images""" def dry_run(self): """Perform environment checks""" self.cloud_image.dry_run()
def __init__(self, working_dir, input_disk_path): super().__init__(working_dir, input_disk_path) self.disk = AzureDisk(input_disk_path, working_dir) self.metadata = CloudImageMetadata() self.compute_client = None self.image_name = None
class AzureDisk(BaseDisk): """ Manage Azure disk """ # pylint: disable=too-many-instance-attributes def __init__(self, input_disk_path, working_dir): """Initialize azure disk object.""" # First initialize the super class. super().__init__(input_disk_path, working_dir) self.uploaded_disk_url = None self.connection_string = get_config_value('AZURE_STORAGE_CONNECTION_STRING') self.container_name = get_config_value('AZURE_STORAGE_CONTAINER_NAME') self.blob = None self.progress_cb_lu = 0 self.metadata = CloudImageMetadata() def clean_up(self): """Clean-up the uploaded disk after image generation.""" def set_uploaded_disk_name(self, disk_name): """Set the uploaded disk name""" # As Azure disk takes its name from the image-name (unlike other clouds where # the disk-name are auto-generated during disk extraction), append disk extension # to the uploaded disk name. self.uploaded_disk_name = disk_name + '.vhd' LOGGER.info("The uploaded disk name is '%s'.", self.uploaded_disk_name) def extract(self): """Extract the vhd disk out of tar.gz.""" self.disk_to_upload = BaseDisk.decompress(self.input_disk_path, '.vhd', self.working_dir) LOGGER.info("Azure disk_to_upload = '%s'", self.disk_to_upload) def _get_tags(self): tags = CloudImageTags(self.metadata) tags.title_case_keys() return tags.get() def _progress_cb(self, byte_up, byte_total): sec = int(time()) # No update within 10 second interval if sec-self.progress_cb_lu > 10: self.progress_cb_lu = sec byte_up //= (1<<20) byte_total //= (1<<20) LOGGER.info('Uploaded %d MB of total %d MB', byte_up, byte_total) def upload(self): """ Upload a F5 BIG-IP VE image to provided container """ def upload_azure(): with open(self.disk_to_upload,'rb') as vhd_file: self.blob.upload_blob( vhd_file.read(), blob_type="PageBlob", metadata=self._get_tags() ) def _upload_impl(): """ Azure blob upload implementation """ timeout = int(get_config_value('AZURE_BLOB_UPLOAD_TIMEOUT')) try: self.connection_string = get_config_value('AZURE_STORAGE_CONNECTION_STRING') LOGGER.info("create blob client") self.blob = BlobClient.from_connection_string( conn_str=self.connection_string, container_name=self.container_name, blob_name=self.uploaded_disk_name, connection_timeout=timeout ) LOGGER.info(self._get_tags()) nonlocal upload_azure upload_azure_p = Process(target=upload_azure) upload_azure_p.start() limit = int(timeout/10) for _ in range(limit): if not upload_azure_p.is_alive(): break sleep(10) os.write(1, b".") else: raise TimeoutError LOGGER.info(self.blob.get_blob_properties()) local_blob_size = os.stat(self.disk_to_upload).st_size uploaded_blob_size = self.blob.get_blob_properties().get("size") LOGGER.info("uploaded blob size: %s and local blob_size: %s", \ str(uploaded_blob_size), str(local_blob_size)) if uploaded_blob_size != local_blob_size: return False except AzureMissingResourceHttpError: LOGGER.error("Exception during uploading %s", self.disk_to_upload) return False except AzureException: LOGGER.error("Exception during uploading %s", self.disk_to_upload) return False except TimeoutError: LOGGER.error("Timeout while uploading") return False self.uploaded_disk_url = self.blob.url # save uploaded disk in artifacts dir json file vhd_url_json = {"vhd_url": self.uploaded_disk_url} artifacts_dir = get_config_value("ARTIFACTS_DIR") with open(artifacts_dir + "/vhd_url.json", "w") as vhd_url_json_file: json.dump(vhd_url_json, vhd_url_json_file) # insert file with vhd url self.metadata.set(self.__class__.__name__, 'vhd_url', self.uploaded_disk_url) self.metadata.set(self.__class__.__name__, 'image_id', self.uploaded_disk_name) LOGGER.info('Uploaded disk url is: %s', self.uploaded_disk_url) return True retrier = Retrier(_upload_impl) retrier.tries = int(get_config_value('AZURE_BLOB_UPLOAD_COMPLETED_RETRY_COUNT')) retrier.delay = int(get_config_value('AZURE_BLOB_UPLOAD_COMPLETED_RETRY_DELAY')) LOGGER.info("Waiting for blob %s to be uploaded.", self.disk_to_upload) if retrier.execute(): LOGGER.info("blob [%s] is ready.", self.disk_to_upload) return True LOGGER.error("blob [%s] was still not ready after checking [%d] times!", self.disk_to_upload, retrier.tries) raise RuntimeError("Runtime Error Occured during Azure Disk Upload")