def load_files(self, metadata_files): """Load metadata from files""" LOGGER.debug('Load metadata files...') for metadata_file in metadata_files: LOGGER.debug('Loading %s...', metadata_file) if not os.path.exists(metadata_file): raise ValueError( 'No such file or directory: {}'.format(metadata_file)) if os.stat(metadata_file).st_size == 0: raise ValueError( 'Empty metadata file: {}'.format(metadata_file)) with open(metadata_file) as metadata_contents: new_metadata = json.load(metadata_contents) if 'build_source' not in new_metadata: raise ValueError( 'build_source field not found in {}'.format( new_metadata)) # Each build_source can have only one set of metadata build_source = new_metadata['build_source'] if build_source in self.metadata: raise ValueError( 'Source {} already loaded'.format(build_source)) # Remove any empty strings self.metadata[build_source] = \ {k: v for k, v in new_metadata.items() if v != u''} LOGGER.trace('metadata:%s', json.dumps(self.metadata, indent=4, sort_keys=True))
def transform_values(self, to_lower=False, disallowed_regex='[^a-zA-Z0-9-]', replacement_char='-'): """Transform data values""" LOGGER.debug('Transform metadata values') if disallowed_regex is not None and disallowed_regex != '': try: re.compile(disallowed_regex) except re.error as exc: raise ValueError('disallowed_regex is invalid: {}'.format( str(exc))) from exc if replacement_char is None or replacement_char == '': raise ValueError( 'Replacement character is required for disallowed_regex') for key, val in self.metadata.items(): # convert values to lower as requested if to_lower: val = val.lower() # substitute replacement character as requested if disallowed_regex is not None and disallowed_regex != '': val = re.sub(disallowed_regex, replacement_char, val) self.metadata[key] = val LOGGER.trace('metadata:%s', json.dumps(self.metadata, indent=4, sort_keys=True))
def _is_snapshot_ready(): """Awaits the import operation represented by the import_task_id to reach 'completed' status.""" try: LOGGER.trace("Querying the status of import-task [%s].", import_task_id) response = \ self.ec2_client.describe_import_snapshot_tasks( ImportTaskIds=[import_task_id]) if not response: raise RuntimeError( "describe_import_snapshot_tasks() returned none response!" ) LOGGER.trace( "Response from describe_import_snapshot_tasks => '%s'", response) task_status = response['ImportSnapshotTasks'][0][ 'SnapshotTaskDetail']['Status'] if task_status == 'error': # Print the response before raising an exception. LOGGER.debug( "describe_import_snapshot_tasks() response for [%s] => [%s]", import_task_id, response) raise RuntimeError( "import-snapshot task [{}] in unrecoverable 'error' state." .format(import_task_id)) return task_status == 'completed' except ClientError as client_error: LOGGER.exception(client_error) raise RuntimeError( "describe_import_snapshot_tasks() failed for [{}]!".format( import_task_id)) from client_error
def share_image(self): """Reads a list of AWS accounts and shares the AMI with each of those accounts.""" share_account_ids = get_list_from_config_yaml('AWS_IMAGE_SHARE_ACCOUNT_IDS') if share_account_ids: LOGGER.info("Share the AMI with multiple AWS accounts.") for dest_account_id in share_account_ids: try: LOGGER.info('Sharing image with account-id: %s', dest_account_id) # Share the image with the destination account response = self.ec2_client.modify_image_attribute( ImageId=self.image_id, Attribute='launchPermission', OperationType='add', UserIds=[str(dest_account_id)] ) LOGGER.trace("image.modify_attribute response => %s", response) except ClientError as client_error: LOGGER.exception(client_error) # Log the error around malformed Account-id and move on. if client_error.response['Error']['Code'] == 'InvalidAMIAttributeItemValue': LOGGER.error('Malformed account-id: %s', dest_account_id) else: # Any other type of error can be irrecoverable and might # point to a deeper malaise. raise RuntimeError('aws IMAGE was not shared with other accounts') \ from client_error # Acknowledge all the account-ids that the image was shared with. self.is_share_image_succeeded(share_account_ids) else: LOGGER.info("No account IDs found for sharing AMI")
def _command_key_values_to_dict(command, regex): """Runs a command in a subprocess, searches the output of the command for key/value pairs using the specified regex, and returns a dictionary containing those pairs""" dictionary = {} LOGGER.debug("Searching for version information using command: %s", command) try: lines = check_output(command.split(), universal_newlines=True, stderr=STDOUT).split('\n') except FileNotFoundError: LOGGER.warning( "Command [%s] not found on system. Unable to check version!", command) return dictionary except CalledProcessError as error: LOGGER.warning( "Skipping version information since command [%s] returned with error: %s", command, error.output) return dictionary for line in lines: LOGGER.trace("Regex search string: %s", regex) LOGGER.trace("Regex search line: %s", line) search = re.search(regex, line) if search: LOGGER.trace("Regex succeeded") dictionary[search.group(1)] = search.group(2) else: LOGGER.trace("Regex failed") LOGGER.trace("Completed dictionary: %s", dictionary) return dictionary
def is_share_image_succeeded(self, share_account_ids): """Helper utility for share_image() that goes through the list of share_account_ids and confirms that the image was shared with all accounts. The function logs any error during its execution without propagating it up.""" try: LOGGER.info("Checking which accounts were added for sharing this AMI") image_launch_perms = self.ec2_client.describe_image_attribute( ImageId=self.image_id, Attribute='launchPermission', DryRun=False ) LOGGER.trace("image.describe_attribute() response => %s", image_launch_perms) except ClientError as client_error: # Simply log the exception without propagating it. LOGGER.exception(client_error) return False # Create a list of account IDs that has launch permission launch_permission_accounts = [] for each in image_launch_perms['LaunchPermissions']: launch_permission_accounts.append(each['UserId']) counter = 0 # Check which accounts were added for sharing this AMI for account_id in share_account_ids: if str(account_id) in launch_permission_accounts: LOGGER.info("The AMI was successfully shared with account: %s", account_id) counter += 1 else: LOGGER.warning("The AMI was not shared with account: %s", account_id) # Confirm that the number of accounts in share_account_ids and image's # 'LaunchPermissions' are matching. return counter == len(share_account_ids)
def create_snapshot(self): """Creates a snapshot from the uploaded s3_disk.""" try: description = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '--BIGIP-Volume-From-' description += self.s3_disk LOGGER.info("Importing the disk [s3://%s/%s] as a snapshot in AWS.", self.s3_bucket, self.s3_disk) response = self.ec2_client.import_snapshot(Description=description, DiskContainer={ "Description": description, "Format": "vmdk", "UserBucket": { "S3Bucket": self.s3_bucket, "S3Key": self.s3_disk } }) LOGGER.trace("import_snapshot() Response => '%s'", response) self.import_task_id = response['ImportTaskId'] LOGGER.info("TaskId for the import_snapshot() operation => [%s]", self.import_task_id) # Wait for the snapshot import to complete. self.is_snapshot_ready(self.import_task_id) # As the import operation successfully completed, reset it back to None # to avoid trying to cancel a completed import-task during clean-up. self.import_task_id = None # Tag the snapshot self.create_tags() except RuntimeError as runtime_error: LOGGER.exception(runtime_error) raise
def filter(self, platform=None): """Filter metadata using config files.""" self.__load_config_attribute_keys(platform) # Build metadata by walking config data and grabbing appropriate metadata self.metadata = {} LOGGER.debug('Filter metadata using config attribute keys') for source_key, source_attribute_keys in self.config_attribute_keys.items( ): LOGGER.debug('Add attributes for source %s', source_key) if source_key not in self.all_metadata: raise ValueError( 'Metadata source_key:{} not found'.format(source_key)) for attribute_key in source_attribute_keys: LOGGER.trace(' Add attribute for key %s:%s', source_key, attribute_key) if attribute_key not in self.all_metadata[source_key]: raise ValueError('Metadata for key {}:{} not found'.format( source_key, attribute_key)) self.metadata[attribute_key] = self.all_metadata[source_key][ attribute_key] LOGGER.trace('metadata:%s', json.dumps(self.metadata, indent=4, sort_keys=True))
def create_image(self, image_name): """Create image implementation for AWS""" # image name must be unique self.delete_old_image(image_name) #start image creation LOGGER.info('Started creation of image %s at %s', image_name, datetime.datetime.now().strftime('%H:%M:%S')) start_time = time() try: response = self.ec2_client.register_image( Architecture="x86_64", BlockDeviceMappings=[ { "DeviceName": AWSImage.AWS_IMAGE_ROOT_VOLUME, "Ebs": { "DeleteOnTermination": True, "SnapshotId": self.snapshot.snapshot_id, "VolumeType": "gp2" } } ], EnaSupport=True, Description=image_name, Name=image_name, RootDeviceName=AWSImage.AWS_IMAGE_ROOT_VOLUME, SriovNetSupport="simple", VirtualizationType="hvm" ) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('register_image failed for image\'{}\'!'.format(image_name)) \ from botocore_exception # get image id try: LOGGER.trace("register_image() response: %s", response) self.image_id = response['ImageId'] except KeyError as key_error: LOGGER.exception(key_error) raise RuntimeError('could not find \'ImageId\' key for image {} '.format(image_name) + 'in create_image response: {}'.format(response)) from key_error LOGGER.info('Image id: %s', self.image_id) # save image id in artifacts dir json file save_image_id(self.image_id) # wait till the end of the image creation self.wait_for_image_availability() LOGGER.info('Creation of %s image took %d seconds', self.image_id, time() - start_time) LOGGER.info('Tagging %s as the image_id.', self.image_id) self.metadata.set(self.__class__.__name__, 'image_id', self.image_id) # add tags to the image self.create_tags()
def to_file(self, build_info_file_path): """Output build info as pre-formatted JSON string to file at specified path""" LOGGER.debug("Writing build info to specified file as a JSON string") output = self.to_json() LOGGER.trace("build_info: %s", output) Path(dirname(build_info_file_path)).mkdir(parents=True, exist_ok=True) with open(build_info_file_path, 'w') as output_file: LOGGER.trace("output_file: %s", build_info_file_path) output_file.writelines(output) LOGGER.debug("Wrote build info to [%s]", build_info_file_path)
def find_image(self, image_name): """ Find image by name. Return response""" try: response = self.ec2_client.describe_images( Filters=[{'Name': 'name', 'Values':[image_name]}]) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('describe_images failed for image \'{}\' !'.format(image_name)) \ from botocore_exception LOGGER.trace('describe_images response for image %s: %s', image_name, response) return response
def create_tags(self): """ Create tags for snapshot. Tags are fetched from metadata. """ snapshot_tags = self.get_snapshot_tag_metadata() tags_to_add = [] for tag in snapshot_tags: tags_to_add.append({'Key': tag, 'Value': snapshot_tags[tag]}) try: response = self.ec2_client.create_tags(Resources=[self.snapshot_id], Tags=tags_to_add) except (ClientError, ParamValidationError) as botocore_exception: LOGGER.exception(botocore_exception) raise RuntimeError('create_tags failed for snapshot\'{}\'!\n'.format(self.snapshot_id)) LOGGER.trace('create_tags response for snapshot %s: %s', self.snapshot_id, response)
def __init__(self, ec2_client, s3_bucket, s3_disk): self.s3_bucket = s3_bucket self.s3_disk = s3_disk # ec2_client object to perform various EC2 operations. self.ec2_client = ec2_client LOGGER.trace("self.s3_disk = '%s'", self.s3_disk) LOGGER.trace("self.s3_bucket = '%s'", self.s3_bucket) # Snapshot created from the uploaded 's3_disk'. self.snapshot_id = None # Import-task-id for the disk import operation. self.import_task_id = None
def register_image(self, skip_post=False, timeout=60.0): """Register image.""" # Check for URL cir_url = get_config_value('IMAGE_REGISTRATION_URL') if (cir_url is None) and (not skip_post): LOGGER.trace( 'IMAGE_REGISTRATION_URL is not defined. Skip image registration.' ) return # Format data metadata = copy.deepcopy(self.metadata) self.registration_data = {} # Azure supports both ASM and ARM models. We are using ARM now. if ('platform' in metadata) and (metadata['platform'] == 'azure'): metadata['platform'] = 'azurerm' # These metadata attributes are used as keys in the registry for key in ['platform', 'image_id', 'image_name']: # special case mapping for register API platform -> cloud if key not in metadata: raise ValueError( '{} attribute missing from metadata'.format(key)) if key == 'platform': self.registration_data['cloud'] = str(metadata[key]) else: self.registration_data[key] = str(metadata[key]) del metadata[key] # Add bundle attribute to support legacy cloud BVT metadata['bundle'] = self.get_bundle(metadata) # All other metadata attributes are considered registry attributes self.registration_data['attributes'] = json.dumps(metadata, sort_keys=True) if skip_post: LOGGER.info('skip_post flag is set. Skip image registration.') LOGGER.trace('Registration data:%s', self.registration_data) return # Register image self.post_to_url(cir_url, timeout)
def __load_config_attribute_keys(self, platform=None): """Load config attributes""" # Load attributes to include in metadata LOGGER.debug('Load config files.') # Always use 'all' and add optional platform self.config_attribute_keys = {} platforms = ['all'] if platform is not None: platforms.append(platform) # Load config files for config_file in self.config_files: LOGGER.debug('Loading %s...', config_file) # Check for empty file if os.stat(config_file).st_size == 0: raise ValueError('Empty config file: {}'.format(config_file)) with open(config_file) as config: new_config = yaml.safe_load(config) # Get build sources for all platforms for config_platform in platforms: # If config file has platform def, add build sources if config_platform in new_config: for build_source in new_config[config_platform]: # Create empty list before adding attribute keys if build_source not in self.config_attribute_keys: self.config_attribute_keys[build_source] = [] self.config_attribute_keys[build_source] += \ new_config[config_platform][build_source] # Attributes must be unique. Check if attributes already exist for any source check_keys = [] for build_source in list(self.config_attribute_keys.keys()): for attribute in self.config_attribute_keys[build_source]: if attribute not in check_keys: check_keys.append(attribute) else: raise ValueError( 'Duplicate attribute {} found in config'.format( attribute)) LOGGER.trace('config_attribute_keys:%s', self.config_attribute_keys)
def title_case_keys(self): """Transform keys to TitleCase. Note if words in a key aren't properly TitleCased or broken up, this won't fix that (e.g. version_jobid is transformed to VersionJobid rather than VersionJobId).""" LOGGER.debug('Transform keys to TitleCase') # Use a copy to avoid changing the data structure that is being iterated over metadata = copy.deepcopy(self.metadata) for key in metadata: # Replace non-alphanumeric with spaces to prepare to capitalize first char of each word new_key = ''.join(c if c.isalnum() else ' ' for c in key) # Capitalize first char of each word new_key = ''.join(word.title() for word in new_key.split()) # Replace existing key with TitleCase key LOGGER.trace('Tranform key %s to %s', key, new_key) self.metadata[new_key] = self.metadata.pop(key) LOGGER.trace('metadata:%s', json.dumps(self.metadata, indent=4, sort_keys=True))
def read_injected_files(top_call_dir, overall_dest_dir): """ Copy file that need to be injected to temporary location, which will be accessible during post-install. Two mandatory arguments: a path from where build-image was called a path to initrd directory that will be available during post_install """ # location used by post-install, should be created only if there are files to inject injected_files = 'etc/injected_files' # location used by post-install overall_dest_dir = overall_dest_dir + '/' + injected_files LOGGER.info('temporary location for injected files: %s', overall_dest_dir) # include user-specified files files_to_inject = get_list_from_config_yaml('UPDATE_IMAGE_FILES') # add build_info.json prep_build_info_for_injection(files_to_inject) # each injected file directory to be stored in a separate directory "file<number>" count = 0 LOGGER.trace("files_to_inject: %s", files_to_inject) for file in files_to_inject: LOGGER.trace("file: %s", file) src = extract_single_worded_key(file, 'source') if src[0] != '/' and src[0] != '~': # make it an absolute path src = top_call_dir + '/' + src src = abspath(realpath(expanduser(src))) dest = extract_single_worded_key(file, 'destination') LOGGER.info('inject %s to temporary location %s', src, dest) file_holder = overall_dest_dir + '/file' + str(count) + '/' # copy source to "src" # source file name does not need to be preserved; # it will be copied to destination path on BIG-IP source_holder = file_holder + 'src' if isfile(src): Path(file_holder).mkdir(parents=True, exist_ok=True) copy2(src, source_holder) elif isdir(src): copytree(src, source_holder) else: raise RuntimeError( '\'{}\' is neither a file nor a directory, cannot inject it!'. format(src)) # store destination if dest[0] != '/': raise RuntimeError( 'injected file destination \'{}\' must be an absolute path!'. format(dest)) with open(file_holder + 'dest', 'w') as dest_holder: print("{}".format(dest), file=dest_holder) count += 1 # end of for loop LOGGER.debug('leaving %s', basename(__file__)) return 0
def read_injected_files(overall_dest_dir): """ Copy file that need to be injected to temporary location, which will be accessible during post-install. One mandatory argument: a path to initrd directory that will be available during post_install """ artifacts_dir = get_config_value("ARTIFACTS_DIR") # location used by post-install, should be created only if there are files to inject injected_files = 'etc/injected_files' # location used by post-install overall_dest_dir = overall_dest_dir + '/' + injected_files LOGGER.info('temporary location for injected files: %s', overall_dest_dir) # include user-specified files files_to_inject = get_list_from_config_yaml('UPDATE_IMAGE_FILES') # include information about installed software on the build machine build_info_file_name = "build_info.json" build_info_source = artifacts_dir + "/" + build_info_file_name build_info_destination = "/" + build_info_file_name files_to_inject.append({ 'source': build_info_source, 'destination': build_info_destination }) build_info = BuildInfo() build_info.to_file(build_info_source) # each injected file directory to be stored in a separate directory "file<number>" count = 0 LOGGER.trace("files_to_inject: %s", files_to_inject) for file in files_to_inject: LOGGER.trace("file: %s", file) src = extract_single_worded_key(file, 'source') dest = extract_single_worded_key(file, 'destination') LOGGER.info('inject %s to temporary location %s', src, dest) file_holder = overall_dest_dir + '/file' + str(count) + '/' # copy source to "src" # source file name does not need to be preserved; # it will be copied to destination path on BIG-IP source_holder = file_holder + 'src' if isfile(src): Path(file_holder).mkdir(parents=True, exist_ok=True) copy2(src, source_holder) elif isdir(src): copytree(src, source_holder) else: raise RuntimeError( '\'{}\' is neither a file nor a directory, cannot inject it!'. format(src)) # store destination if dest[0] != '/': raise RuntimeError( 'injected file destination \'{}\' must be an absolute path!'. format(dest)) with open(file_holder + 'dest', 'w') as dest_holder: print("{}".format(dest), file=dest_holder) count += 1 # end of for loop LOGGER.debug('leaving %s', basename(__file__)) return 0
def read_injected_files(top_call_dir, overall_dest_dir): """ Copy files that need to be injected to a temporary location, which will be accessible during post-install. Two mandatory arguments: a path from where build-image was called a path to initrd directory that will be available during post_install """ # location used by post-install, should be created only if there are files to inject injected_files = 'etc/injected_files' # location used by post-install overall_dest_dir = overall_dest_dir + '/' + injected_files LOGGER.info('Temporary location for injected files: \'%s\'', overall_dest_dir) # include user-specified files files_to_inject = get_list_from_config_yaml('UPDATE_IMAGE_FILES') # add build_info.json prep_build_info_for_injection(files_to_inject) # each injected file directory to be stored in a separate directory "file<number>" count = 0 LOGGER.trace("files_to_inject: %s", files_to_inject) for file in files_to_inject: LOGGER.debug('Injecting file: \'%s\'.', file) src = extract_single_worded_key(file, 'source') dest = extract_single_worded_key(file, 'destination') if 'mode' in file: mode = extract_single_worded_key(file, 'mode') else: mode = None LOGGER.info('Copy \'%s\' to a temporary location for \'%s\'.', src, dest) url = src # treat 'src' as a file path and 'url' as a url if src[0] != '/' and src[0] != '~': # make it an absolute path src = top_call_dir + '/' + src src = abspath(realpath(expanduser(src))) file_holder = overall_dest_dir + '/file' + str(count) + '/' # copy source to "src" # source file name does not need to be preserved; # it will be copied to destination path on BIG-IP source_holder = file_holder + 'src' Path(file_holder).mkdir(parents=True, exist_ok=True) if isfile(src): LOGGER.info('Treating \'%s\' as a file for file injection', src) copy2(src, source_holder) elif isdir(src): LOGGER.info('Treating \'%s\' as a directory for file injection', src) copytree(src, source_holder) else: LOGGER.info('Treating \'%s\' as a URL for the file injection', url) download_file(url, source_holder) # store destination if dest[0] != '/': raise RuntimeError( 'injected file destination \'{}\' must be an absolute path!'. format(dest)) with open(file_holder + 'dest', 'w') as dest_holder: print("{}".format(dest), file=dest_holder) # Store mode. Should be a string consisting of one to four octal digits. if mode: LOGGER.debug('Creating mode holder for mode \'%s\'.', mode) mode_pattern = re.compile('^[0-7][0-7]?[0-7]?[0-7]?$') if not mode_pattern.match(mode): raise RuntimeError('Invalid mode \'' + mode + '\', must be a string ' + 'consisting of one to four octal digits.') with open(file_holder + 'mode', 'w') as mode_holder: print("{}".format(mode), file=mode_holder) count += 1 # end of for loop LOGGER.debug('leaving %s', basename(__file__)) return 0