def delete_blob(self): """ Delete the blob corresponding to self.uploaded_disk_name if it exists. """ try: # Populate the bucket if not already. if self.bucket is None: self.init_bucket() if self.uploaded_disk_name is None: raise RuntimeError( "Trying to delete a non-existent uploaded disk.") blob = self.get_blob(self.uploaded_disk_name) if blob is not None: LOGGER.info("Deleting blob '%s'.", self.uploaded_disk_name) self.bucket.delete_blob(self.uploaded_disk_name) blob = self.get_blob(self.uploaded_disk_name) if blob is not None: raise RuntimeError( "Deleting blob '{}' silently failed as it still exists." .format(self.uploaded_disk_name)) except google.cloud.exceptions.NotFound as exception: LOGGER.exception(exception) raise exception except RuntimeError as runtime_exception: raise runtime_exception
def is_image_ready(self, image_name): """Checks if the given image is ready.""" def _is_image_ready(): """Checks if an image with image_name exists and status is READY""" # pylint: disable=no-member request = self.gce_service.images().get( project=self.gce_project_id, image=image_name) result = request.execute() if not result or result['status'] == 'FAILED': raise RuntimeError( "Creation of image [{}] failed!".format(image_name)) return result['status'] == 'READY' retrier = Retrier(_is_image_ready) retrier.tries = int( get_config_value('GCE_IMAGE_CREATE_COMPLETED_RETRY_COUNT')) retrier.delay = int( get_config_value('GCE_IMAGE_CREATE_COMPLETED_RETRY_DELAY')) LOGGER.info("Waiting for image [%s] to be ready.", image_name) try: if retrier.execute(): LOGGER.info("Image [%s] is ready.", image_name) self.metadata.set(self.__class__.__name__, 'image_id', image_name) return True LOGGER.warning( "Image [%s] was still not ready after checking [%d] times!", image_name, retrier.tries) return False except HttpError as exp: LOGGER.exception(exp) return False except RuntimeError as runtime_exception: LOGGER.exception(runtime_exception) return False
def delete_old_image(self, image_name): """ Check if an image with the same name already exists and delete it. This is unlikely to happen unless the image name is specified in the configuration.""" response = self.find_image(image_name) num_images = len(response['Images']) if num_images not in (0, 1, 2): raise RuntimeError('Number of images named {} '.format(image_name) + 'expected to be 0 or 1 (maybe 2, due to AWS replicability issues),' + ' but found {}. '.format(num_images) + '(Should have received InvalidAMIName.Duplicate error during ' + 'the previous image creation). Please delete them manually.') if num_images in (1, 2): try: first_image_id = response['Images'][0]['ImageId'] if num_images == 2: second_image_id = response['Images'][1]['ImageId'] except KeyError as key_error: LOGGER.exception(key_error) raise RuntimeError( 'could not find ImageId key for image {} '.format(image_name) + 'in describe_images response: {}'.format(response)) from key_error LOGGER.info('There is an old image %s named %s, deleting it.', first_image_id, image_name) self.delete_image(first_image_id) if num_images == 2: LOGGER.info('There is an old image %s named %s, deleting it.', second_image_id, image_name) self.delete_image(second_image_id)
def share_image(self): """Reads a list of AWS accounts and shares the AMI with each of those accounts.""" share_account_ids = get_list_from_config_yaml('AWS_IMAGE_SHARE_ACCOUNT_IDS') if share_account_ids: LOGGER.info("Share the AMI with multiple AWS accounts.") for dest_account_id in share_account_ids: try: LOGGER.info('Sharing image with account-id: %s', dest_account_id) # Share the image with the destination account response = self.ec2_client.modify_image_attribute( ImageId=self.image_id, Attribute='launchPermission', OperationType='add', UserIds=[str(dest_account_id)] ) LOGGER.trace("image.modify_attribute response => %s", response) except ClientError as client_error: LOGGER.exception(client_error) # Log the error around malformed Account-id and move on. if client_error.response['Error']['Code'] == 'InvalidAMIAttributeItemValue': LOGGER.error('Malformed account-id: %s', dest_account_id) else: # Any other type of error can be irrecoverable and might # point to a deeper malaise. raise RuntimeError('aws IMAGE was not shared with other accounts') \ from client_error # Acknowledge all the account-ids that the image was shared with. self.is_share_image_succeeded(share_account_ids) else: LOGGER.info("No account IDs found for sharing AMI")
def is_share_image_succeeded(self, share_account_ids): """Helper utility for share_image() that goes through the list of share_account_ids and confirms that the image was shared with all accounts. The function logs any error during its execution without propagating it up.""" response_json = None try: LOGGER.info("Checking which accounts were added for sharing this image") response_json = self.client.describe_image_share_permission(self.image_id) except ServerException as exc: LOGGER.exception(exc) if exc.get_error_code() == 'InvalidImageId.NotFound' and \ exc.get_error_msg().startswith('The specified ImageId does not exist'): raise RuntimeError('InvalidImageId.NotFound: Check if the Image ID exists') raise exc num_accounts = len(response_json['Accounts']['Account']) shared_accounts = [] for each_account in range(num_accounts): account_id = response_json['Accounts']['Account'][each_account]['AliyunId'] shared_accounts.append(int(account_id)) counter = 0 for an_account in share_account_ids: if an_account in shared_accounts: LOGGER.info("The image was successfully shared with account: %s", an_account) counter += 1 else: LOGGER.warning("The image was not shared with account: %s", an_account) # Confirm that the number of accounts in share_account_ids and image's # 'LaunchPermissions' are matching. return counter == len(share_account_ids)
def is_share_image_succeeded(self, share_account_ids): """Helper utility for share_image() that goes through the list of share_account_ids and confirms that the image was shared with all accounts. The function logs any error during its execution without propagating it up.""" try: LOGGER.info("Checking which accounts were added for sharing this AMI") image_launch_perms = self.ec2_client.describe_image_attribute( ImageId=self.image_id, Attribute='launchPermission', DryRun=False ) LOGGER.trace("image.describe_attribute() response => %s", image_launch_perms) except ClientError as client_error: # Simply log the exception without propagating it. LOGGER.exception(client_error) return False # Create a list of account IDs that has launch permission launch_permission_accounts = [] for each in image_launch_perms['LaunchPermissions']: launch_permission_accounts.append(each['UserId']) counter = 0 # Check which accounts were added for sharing this AMI for account_id in share_account_ids: if str(account_id) in launch_permission_accounts: LOGGER.info("The AMI was successfully shared with account: %s", account_id) counter += 1 else: LOGGER.warning("The AMI was not shared with account: %s", account_id) # Confirm that the number of accounts in share_account_ids and image's # 'LaunchPermissions' are matching. return counter == len(share_account_ids)
def upload(self): """ Upload tar.gz stored at self.disk_to_upload to Google storage """ try: # Populate the bucket if not already. if self.bucket is None: self.init_bucket() # form blob name prefix = datetime.datetime.now().strftime('%Y%m%d') + '/' self.uploaded_disk_name = prefix + BaseDisk.decorate_disk_name(self.disk_to_upload) # delete the blob if it exists self.delete_blob() # create blob blob = self.bucket.blob(self.uploaded_disk_name) if blob is None: raise RuntimeError("Factory constructor for blob '{}' failed." .format(self.uploaded_disk_name)) # upload blob LOGGER.info("Started to upload '%s' at '%s'.", self.uploaded_disk_name, datetime.datetime.now().strftime('%H:%M:%S')) blob.upload_from_filename(self.disk_to_upload) LOGGER.info("Finished to upload '%s' at '%s'.", self.uploaded_disk_name, datetime.datetime.now().strftime('%H:%M:%S')) if not blob.exists(): raise RuntimeError("Uploading blob '{}' failed.".format(self.uploaded_disk_name)) except RuntimeError as exception: LOGGER.exception(exception) raise exception
def create_snapshot(self): """Creates a snapshot from the uploaded s3_disk.""" try: description = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '--BIGIP-Volume-From-' description += self.s3_disk LOGGER.info("Importing the disk [s3://%s/%s] as a snapshot in AWS.", self.s3_bucket, self.s3_disk) response = self.ec2_client.import_snapshot(Description=description, DiskContainer={ "Description": description, "Format": "vmdk", "UserBucket": { "S3Bucket": self.s3_bucket, "S3Key": self.s3_disk } }) LOGGER.trace("import_snapshot() Response => '%s'", response) self.import_task_id = response['ImportTaskId'] LOGGER.info("TaskId for the import_snapshot() operation => [%s]", self.import_task_id) # Wait for the snapshot import to complete. self.is_snapshot_ready(self.import_task_id) # As the import operation successfully completed, reset it back to None # to avoid trying to cancel a completed import-task during clean-up. self.import_task_id = None # Tag the snapshot self.create_tags() except RuntimeError as runtime_error: LOGGER.exception(runtime_error) raise
def prepare(self, seed_image_name='', user_image_name=''): """Main controller""" try: self.set_image_name(seed_image_name, user_image_name) LOGGER.info("Starting prepare cloud image '%s'.", self.image_name) self.cloud_image.set_uploaded_disk_name(self.image_name) pipeline_build = os.getenv('CI') is not None self.initialize_image_metadata(self.artifacts_dir, pipeline_build) self.cloud_image.extract_disk() self.cloud_image.upload_disk() self.cloud_image.prep_disk() self.metadata.set(self.__class__.__name__, 'build_operation', 'create') self.cloud_image.create_image(self.image_name) build_time = time.time() - self.start_time self.metadata.set(self.__class__.__name__, 'build_time', str(timedelta(seconds=build_time))) self.status = 'success' self.metadata.set(self.__class__.__name__, 'status', self.status) self.cloud_image.share_image() self.create_metadata() self.register_image() self.create_report() LOGGER.info("Finished prepare cloud image '%s'.", self.image_name) except BaseException as base_exception: LOGGER.exception(base_exception) raise base_exception
def download_file(url, dest_file): """ Download from url to a local file. Throws exceptions with wording specific to the file injection. Assumes that the directory containing the destination file already exists. """ verify_tls = bool(get_config_value("IGNORE_DOWNLOAD_URL_TLS") is None) try: remote_file = requests.get(url, verify=verify_tls, timeout=60) except requests.exceptions.SSLError as exc: LOGGER.exception(exc) raise RuntimeError( 'Cannot access \'{}\' due to TLS problems! '.format(url) + 'Consider abandoning TLS verification by usage of ' + '\'IGNORE_DOWNLOAD_URL_TLS\' parameter.') except requests.exceptions.RequestException as exc: LOGGER.exception(exc) raise RuntimeError( '\'{}\' is neither a file nor a directory nor a valid url, cannot inject it!' .format(url)) if remote_file.status_code != 200: LOGGER.info('requests.get response status: %s', remote_file.status_code) LOGGER.info('requests.get response headers: %s', remote_file.headers) raise RuntimeError( 'URL \'{}\' did not return content, cannot inject it!'.format(url)) open(dest_file, 'wb').write(remote_file.content)
def _is_snapshot_ready(): """Awaits the import operation represented by the import_task_id to reach 'completed' status.""" try: LOGGER.trace("Querying the status of import-task [%s].", import_task_id) response = \ self.ec2_client.describe_import_snapshot_tasks( ImportTaskIds=[import_task_id]) if not response: raise RuntimeError( "describe_import_snapshot_tasks() returned none response!" ) LOGGER.trace( "Response from describe_import_snapshot_tasks => '%s'", response) task_status = response['ImportSnapshotTasks'][0][ 'SnapshotTaskDetail']['Status'] if task_status == 'error': # Print the response before raising an exception. LOGGER.debug( "describe_import_snapshot_tasks() response for [%s] => [%s]", import_task_id, response) raise RuntimeError( "import-snapshot task [{}] in unrecoverable 'error' state." .format(import_task_id)) return task_status == 'completed' except ClientError as client_error: LOGGER.exception(client_error) raise RuntimeError( "describe_import_snapshot_tasks() failed for [{}]!".format( import_task_id)) from client_error
def insert_image(self, image_name): """Create image in GCE and then check for status = READY""" bucket_name = get_config_value('GCE_BUCKET') image_body = { "name": image_name, "rawDisk": { # In the following line the bucket name along with blob name is required "source": "https://storage.googleapis.com/{}/{}".format( bucket_name, self.disk.uploaded_disk_name) } } try: # pylint: disable=no-member request = self.gce_service.images().insert( project=self.gce_project_id, body=image_body) result = request.execute() except HttpError as exp: LOGGER.exception(exp) raise exp if not result: return False LOGGER.debug("Image creation response: '%s'", result) return self.is_image_ready(image_name)
def _resumable_upload(): self.uploaded_disk_name = 'bakery-' + os.path.basename(self.disk_to_upload) + '-' + \ ''.join(random.choices(string.digits, k=6)) AlibabaDisk.iter += 1 LOGGER.info('Upload iteration number %d', AlibabaDisk.iter) LOGGER.info('Uploading %s as %s', self.disk_to_upload, self.uploaded_disk_name) start_time = time.time() time.sleep(1) result = False try: resumable_store = oss2.resumable.ResumableStore( root=self.working_dir) oss2.resumable_upload(self.bucket, self.uploaded_disk_name, self.disk_to_upload, store=resumable_store, num_threads=number_of_threads) result = True except FileNotFoundError as exc: LOGGER.exception(exc) raise RuntimeError('Could not find file to upload: {}'.format( self.disk_to_upload)) except oss2.exceptions.NoSuchUpload as exc: LOGGER.error('Upload failed. UploadId: %s', exc.details['UploadId']) LOGGER.exception(exc) LOGGER.info('Iteration %d of upload took %d seconds', AlibabaDisk.iter, time.time() - start_time) if not result: self.upload_cleanup() return result
def share_image(self): """Reads a list of account IDs and shares the image with each of those accounts.""" share_account_ids = get_list_from_config_yaml('ALIBABA_IMAGE_SHARE_ACCOUNT_IDS') if share_account_ids: try: LOGGER.info("Share the image with multiple accounts.") self.client.share_image_with_accounts(self.image_id, share_account_ids) except ServerException as exc: LOGGER.exception(exc) if exc.get_error_code() == 'InvalidAccount.NotFound' and \ exc.get_error_msg().startswith('The specified parameter "AddAccount.n" or ' + '"RemoveAccount.n" does not exist.'): raise RuntimeError('InvalidAccount.NotFound: Check if the account IDs are ' + 'correct') if exc.get_error_code() == 'InvalidImageId.NotFound' and \ exc.get_error_msg().startswith('The specified ImageId does not exist'): raise RuntimeError('InvalidImageId.NotFound: Check if the Image ID exists') raise exc # Acknowledge all the account-ids that the image was shared with. if self.is_share_image_succeeded(share_account_ids): LOGGER.info("Image sharing with other accounts was successful") else: LOGGER.info("No account IDs found for sharing the image")
def is_bucket_exist(self): """Checks if a bucket with self.bucket_name exists in S3.""" try: return self.s3_resource.Bucket( self.bucket_name).creation_date is not None except ClientError as client_error: LOGGER.exception(client_error) return False
def delete_image(self, image_id): """ Delete image by image id. Return response""" try: self.ec2_client.deregister_image(ImageId=image_id) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('deregister_image failed for image \'{}\' !'.format(image_id)) \ from botocore_exception
def create_image(self, image_name): """Create image implementation for AWS""" # image name must be unique self.delete_old_image(image_name) #start image creation LOGGER.info('Started creation of image %s at %s', image_name, datetime.datetime.now().strftime('%H:%M:%S')) start_time = time() try: response = self.ec2_client.register_image( Architecture="x86_64", BlockDeviceMappings=[ { "DeviceName": AWSImage.AWS_IMAGE_ROOT_VOLUME, "Ebs": { "DeleteOnTermination": True, "SnapshotId": self.snapshot.snapshot_id, "VolumeType": "gp2" } } ], EnaSupport=True, Description=image_name, Name=image_name, RootDeviceName=AWSImage.AWS_IMAGE_ROOT_VOLUME, SriovNetSupport="simple", VirtualizationType="hvm" ) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('register_image failed for image\'{}\'!'.format(image_name)) \ from botocore_exception # get image id try: LOGGER.trace("register_image() response: %s", response) self.image_id = response['ImageId'] except KeyError as key_error: LOGGER.exception(key_error) raise RuntimeError('could not find \'ImageId\' key for image {} '.format(image_name) + 'in create_image response: {}'.format(response)) from key_error LOGGER.info('Image id: %s', self.image_id) # save image id in artifacts dir json file save_image_id(self.image_id) # wait till the end of the image creation self.wait_for_image_availability() LOGGER.info('Creation of %s image took %d seconds', self.image_id, time() - start_time) LOGGER.info('Tagging %s as the image_id.', self.image_id) self.metadata.set(self.__class__.__name__, 'image_id', self.image_id) # add tags to the image self.create_tags()
def __init__(self, artifacts_dir, cloud_type, image_disk_path, should_clean=True): self.start_time = time.time() self.artifacts_dir = artifacts_dir self.cloud_type = cloud_type self.image_name = None self.image_disk_path = image_disk_path self.metadata = None self.status = 'failure' self.transformed_image_name = None self.working_dir = None self.should_clean = should_clean self.cloud_image = None if not os.path.isdir(artifacts_dir): raise ValueError( "Missing or invalid artifacts directory '{}'.".format( artifacts_dir)) if not os.path.isfile(image_disk_path): raise ValueError( "Missing image disk '{}'.".format(image_disk_path)) # Create a working directory under the artifacts dir to temporarily store # various build constructs and files. self.create_working_dir(artifacts_dir) try: # Factory (could be a separate object) # pylint: disable=import-outside-toplevel if cloud_type == 'alibaba': from image.alibaba_image import AlibabaImage self.cloud_image = AlibabaImage(self.working_dir, self.image_disk_path) elif cloud_type == 'aws': from image.aws_image import AWSImage self.cloud_image = AWSImage(self.working_dir, self.image_disk_path) elif cloud_type == 'azure': from image.azure_image import AzureImage self.cloud_image = AzureImage(self.working_dir, self.image_disk_path) elif cloud_type == 'gce': from image.google_image import GoogleImage self.cloud_image = GoogleImage(self.working_dir, self.image_disk_path) else: raise ValueError( 'Unexpected cloud type: {}'.format(cloud_type)) # pylint: enable=import-outside-toplevel self.cloud_image_name = self.image_name_factory(cloud_type) except BaseException as base_exception: LOGGER.exception(base_exception) raise base_exception
def is_disk_exist(self, disk_name): """Checks if the given disk_name exists in the bucket in S3""" try: bucket = self.get_bucket() if bucket is not None: bucket_objs = list(bucket.objects.filter(Prefix=disk_name)) return bucket_objs and bucket_objs[0].key == disk_name except ClientError as client_error: LOGGER.exception(client_error) return False
def clean_up(self): """Clean-up.""" try: if self.bucket_name is not None and self.uploaded_disk_name is not None: LOGGER.debug("Deleting '%s' from the bucket '%s'.", self.uploaded_disk_name, self.bucket_name) self.delete_uploaded_disk(self.uploaded_disk_name) except ClientError as client_error: # Log the exception without propagating it further. LOGGER.exception(client_error)
def main(): """main command handler""" parser = argparse.ArgumentParser(description='Prepare a cloud image from a virtual disk') parser.add_argument('-a', '--artifacts-dir', required=True, help='Absolute path to the artifacts directory') parser.add_argument('-c', '--check-name', action="store_true", help='Check cloud image name') parser.add_argument('-i', '--input', required=True, help='Absolute path to the input virtual disk') parser.add_argument('-p', '--platform', required=True, help='The cloud type (i.e. aws, gce, azure, alibaba)') parser.add_argument('-s', '--seed-image-name', default='', help='Use supplied autogenerated seed cloud image name') parser.add_argument('-u', '--user-image-name', default='', help='Use user-supplied cloud image name') args = parser.parse_args() # Check either seed or user cloud image name was provided if (args.seed_image_name == '' and args.user_image_name == '') or \ (args.seed_image_name != '' and args.user_image_name != ''): raise Exception('You must provide either --seed-image-name or --user-image-name') # create log handler for the global LOGGER create_log_handler() if args.check_name: # Check name if args.user_image_name == '': raise Exception('--check-name can only be used with --user-image-name') ImageController.check_valid_name(args.platform, args.user_image_name) else: result = False try: # Prepare image image_controller = ImageController(args.artifacts_dir, args.platform, args.input) image_controller.prepare(args.seed_image_name, args.user_image_name) # If execution came so far, all is well. result = True except RuntimeError as runtime_exce: LOGGER.exception(runtime_exce) finally: # Clean-up image controller and other internal constructs it created. image_controller.clean_up() if result is True: LOGGER.info("SUCCESS: Image generation completed.") else: LOGGER.warning("FAILURE: Check the log file '%s' and fix the problem " "before re-running.", get_config_value('LOG_FILE')) sys.exit(1) sys.exit(0)
def tag_image(self, image_name): """Associate image tags with image""" LOGGER.info('Set image labels.') # Get current labels fingerprint. To avoid/detect conflicts, you must # provide the current label fingerprint (reference) when you request to # set image labels. This fingerprint value is updated whenever labels # are updated and the set labels request will fail if the labels were # updated out of band. try: # pylint: disable=no-member request = self.gce_service.images().get( project=self.gce_project_id, image=image_name) result = request.execute() label_fingerprint = result['labelFingerprint'] except HttpError as exp: LOGGER.error("Exception setting image labels:") LOGGER.exception(exp) return False if not result: return False if label_fingerprint is None or label_fingerprint == '': LOGGER.info('Label fingerprint was empty.') return False cloud_image_tags = CloudImageTags(self.metadata) cloud_image_tags.transform_values(to_lower=True, disallowed_regex='[^a-z0-9-]') image_labels = cloud_image_tags.get() set_labels_body = { "labels": image_labels, "labelFingerprint": label_fingerprint } try: # pylint: disable=no-member request = self.gce_service.images().setLabels( project=self.gce_project_id, resource=image_name, body=set_labels_body) result = request.execute() except HttpError as exp: LOGGER.error("Exception setting image labels:") LOGGER.exception(exp) return False if not result: return False LOGGER.debug("Image set labels response: %s", result) return True
def find_image(self, image_name): """ Find image by name. Return response""" try: response = self.ec2_client.describe_images( Filters=[{'Name': 'name', 'Values':[image_name]}]) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('describe_images failed for image \'{}\' !'.format(image_name)) \ from botocore_exception LOGGER.trace('describe_images response for image %s: %s', image_name, response) return response
def get_blob(self, disk_path): """ Gets the GCE disk blob representing the given disk_path. """ blob = None try: blob = self.bucket.get_blob(disk_path) except google.cloud.exceptions.NotFound as exception: LOGGER.exception(exception) raise exception return blob
def __init__(self, artifacts_dir, cloud_type, image_disk_path, should_clean=True): self.start_time = time.time() self.artifacts_dir = artifacts_dir self.cloud_type = cloud_type self.image_name = None self.image_disk_path = image_disk_path self.metadata = None self.status = 'failure' self.transformed_image_name = None self.working_dir = None self.should_clean = should_clean self.cloud_image = None if not os.path.isdir(artifacts_dir): raise ValueError( "Missing or invalid artifacts directory '{}'.".format( artifacts_dir)) if not is_supported_cloud(cloud_type): raise ValueError("Unexpected cloud '{}'.".format(cloud_type)) if not os.path.isfile(image_disk_path): raise ValueError( "Missing image disk '{}'.".format(image_disk_path)) # Create a working directory under the artifacts dir to temporarily store # various build constructs and files. self.create_working_dir(artifacts_dir) try: # Factory (could be a separate object) if cloud_type == 'alibaba': LOGGER.warning("Unimplemented cloud '%s'.", cloud_type) raise SystemExit(-1) if cloud_type == 'aws': self.cloud_image = AWSImage(self.working_dir, self.image_disk_path) elif cloud_type == 'azure': self.cloud_image = AzureImage(self.working_dir, self.image_disk_path) elif cloud_type == 'gce': self.cloud_image = GoogleImage(self.working_dir, self.image_disk_path) else: raise ValueError('Unexpected cloud type') self.cloud_image_name = self.image_name_factory(cloud_type) except BaseException as base_exception: LOGGER.exception(base_exception) raise base_exception
def create_tags(self): """ Create tags for snapshot. Tags are fetched from metadata. """ snapshot_tags = self.get_snapshot_tag_metadata() tags_to_add = [] for tag in snapshot_tags: tags_to_add.append({'Key': tag, 'Value': snapshot_tags[tag]}) try: response = self.ec2_client.create_tags(Resources=[self.snapshot_id], Tags=tags_to_add) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('create_tags failed for snapshot\'{}\'!\n'.format(self.snapshot_id)) LOGGER.trace('create_tags response for snapshot %s: %s', self.snapshot_id, response)
def is_snapshot_ready(self, import_task_id): """Checks if a snapshot with the given import_task_id exists and its status is 'completed'.""" def _is_snapshot_ready(): """Awaits the import operation represented by the import_task_id to reach 'completed' status.""" try: LOGGER.trace("Querying the status of import-task [%s].", import_task_id) response = \ self.ec2_client.describe_import_snapshot_tasks( ImportTaskIds=[import_task_id]) if not response: raise RuntimeError("describe_import_snapshot_tasks() returned none response!") LOGGER.trace("Response from describe_import_snapshot_tasks => '%s'", response) task_status = response['ImportSnapshotTasks'][0]['SnapshotTaskDetail']['Status'] if task_status == 'error': # Print the response before raising an exception. LOGGER.debug("describe_import_snapshot_tasks() response for [%s] => [%s]", import_task_id, response) raise RuntimeError("import-snapshot task [{}] in unrecoverable 'error' state.". format(import_task_id)) return task_status == 'completed' except ClientError as client_error: LOGGER.exception(client_error) raise RuntimeError("describe_import_snapshot_tasks() failed for [{}]!". format(import_task_id)) retrier = Retrier(_is_snapshot_ready) retrier.tries = int(get_config_value('AWS_IMPORT_SNAPSHOT_TASK_RETRY_COUNT')) retrier.delay = int(get_config_value('AWS_IMPORT_SNAPSHOT_TASK_RETRY_DELAY')) LOGGER.info("Waiting for the import snapshot task [%s] to complete.", import_task_id) try: if retrier.execute(): LOGGER.info("import_snapshot_task [%s] is completed.", import_task_id) # Call it one last time to get the snapshot_id. response = \ self.ec2_client.describe_import_snapshot_tasks( ImportTaskIds=[import_task_id]) self.snapshot_id = \ response['ImportSnapshotTasks'][0]['SnapshotTaskDetail']['SnapshotId'] LOGGER.info("SnapshotID = [%s].", self.snapshot_id) return True LOGGER.warning("import_snapshot_task [%s] didn't complete after checking [%d] times!", import_task_id, retrier.tries) return False except RuntimeError as runtime_exception: LOGGER.exception(runtime_exception) raise
def set_bucket(self): """Return bucket for uploaded files""" access_key = get_config_value('ALIBABA_ACCESS_KEY_ID') secret_key = get_config_value('ALIBABA_ACCESS_KEY_SECRET') auth = oss2.Auth(access_key, secret_key) region = get_config_value('ALIBABA_REGION') bucket_name = get_config_value('ALIBABA_BUCKET') self.bucket = oss2.Bucket(auth, 'https://oss-' + region + '.aliyuncs.com', bucket_name) try: self.bucket.get_bucket_info() except oss2.exceptions.SignatureDoesNotMatch as exc: LOGGER.exception(exc) raise RuntimeError('Bad credentials to get bucket info') except oss2.exceptions.ServerError as exc: if exc.details['Code'] == 'InvalidBucketName': LOGGER.exception(exc) raise RuntimeError('Invalid bucket name: ' + exc.details['BucketName']) LOGGER.exception(exc) raise RuntimeError('Unexpected Alibaba oss server error. ' + 'One of possible errors: invalid credentials.') except oss2.exceptions.RequestError as exc: LOGGER.exception(exc) raise RuntimeError( 'Alibaba oss request error. ' + 'One of possible errors: invalid Alibaba region.')
def clean_up(self): """Clean-up the uploaded disk after image generation.""" # Delete the uploaded disk as it no longer needs to be retained. try: if self.bucket and self.uploaded_disk_name: self.delete_blob() except google.cloud.exceptions.NotFound as exception: # Report the exception without propagating it up to ensure this # doesn't stop the rest of the clean-up. LOGGER.error("Caught exception during '%s' disk deletion.", self.uploaded_disk_name) LOGGER.exception(exception) except RuntimeError as runtime_exception: LOGGER.error("Caught runtime exception during '%s' disk deletion.", self.uploaded_disk_name) LOGGER.exception(runtime_exception)
def main(): """ Wrapper to read user defined values for LV sizes """ # create log handler for the global LOGGER create_log_handler() if len(sys.argv) != 2: LOGGER.error('%s received %s arguments, expected 1', basename(__file__), len(sys.argv) - 1) sys.exit(1) try: read_lv_sizes(sys.argv[1]) except RuntimeError as runtime_exception: LOGGER.exception(runtime_exception) sys.exit(1) sys.exit(0)