def clean_up(self): """Clean-up.""" if self.disk_to_upload is not None and os.path.exists( self.disk_to_upload): LOGGER.debug( "Deleting the local disk that was (or had to be) uploaded:= '%s'", self.disk_to_upload) os.remove(self.disk_to_upload) else: failed_delete_file_msg = "Could not find (and delete) the local disk that " + \ "was (or had to be) uploaded: {}".format(self.disk_to_upload) LOGGER.debug(failed_delete_file_msg) LOGGER.debug("Cleaning up the uploaded disk from Alibaba storage") self.upload_cleanup()
def main(): """main publish telemetry information function""" # create log handler for the global LOGGER create_log_handler() # gather telemetry info build_info_telemetry = BuildInfoTelemetry() LOGGER.debug("telemetry info:") LOGGER.debug(build_info_telemetry.build_info) version = build_info_telemetry.build_info['product']['version'] # Check if specific api key is set, if not use default if environ.get("F5_TEEM_API_KEY") is not None: environ['F5_TEEM_API_ENVIRONMENT'] = "staging" f5_api_key = environ.get("F5_TEEM_API_KEY") else: f5_api_key = 'mmhJU2sCd63BznXAXDh4kxLIyfIMm3Ar' generated_uuid = str(uuid.uuid4()) LOGGER.debug("telemetry UUID: %s", generated_uuid) client_info = { 'name': 'f5-image-generator', 'version': str(version), 'id': generated_uuid } telemetry_client = AnonymousDeviceClient(client_info, api_key=f5_api_key) retrier = Retrier(_publish_telemetry_database, build_info_telemetry, telemetry_client) retrier.tries = int(get_config_value('PUBLISH_TELEMETRY_TASK_RETRY_COUNT')) retrier.delay = int(get_config_value('PUBLISH_TELEMETRY_TASK_RETRY_DELAY')) if retrier.execute(): LOGGER.info("Publishing to telemetry success.") return True LOGGER.info("Publishing to telemetry did not succeed.") sys.exit(0)
def get_python_modules(): """Collect version information for active Python modules""" LOGGER.debug("Collecting version information for active Python modules") python_command = BuildInfo.package_manager_commands["python"] python_modules = _command_key_values_to_dict(*python_command) return python_modules
def create_image(self, image_name): """ Create image implementation for Alibaba """ images_json = self.client.describe_images(None, image_name) if int(images_json['TotalCount']) == 0: LOGGER.debug('No old images named \'%s\' were found', image_name) else: # image names are unique, delete only one image image_id = images_json['Images']['Image'][0]['ImageId'] LOGGER.info('Image \'%s\' already exists, its id is \'%s\', deleting it', image_name, image_id) self.client.delete_image(image_id) # start image creation LOGGER.info('Started creation of image \'%s\' at %s', image_name, datetime.datetime.now().strftime('%H:%M:%S')) start_time = time() imported_image = self.client.import_image(get_config_value('ALIBABA_BUCKET'), self.disk.uploaded_disk_name, image_name) if 'Code' in imported_image.keys(): if imported_image['Code'] == 'InvalidOSSObject.NotFound': raise RuntimeError('ImportImageRequest could not find uloaded disk \'' + image_name + '\'') if imported_image['Code'] == 'InvalidImageName.Duplicated': raise RuntimeError('Image \'' + image_name + '\' still exists, ' + 'should have been removed by this point') if imported_image['Code'] == 'ImageIsImporting': raise RuntimeError('Another image named \'' + image_name + '\' is in the ' + 'process of importing, probably from the previous run. ' + 'Delete it first.') if 'ImageId' not in imported_image.keys() or 'TaskId' not in imported_image.keys(): LOGGER.info('Alibaba response to ImportImageRequest:') LOGGER.info(json.dumps(imported_image, sort_keys=True, indent=4, separators=(',', ': '))) raise RuntimeError('ImageId and/or TaskId were not found in the response ' + 'cannot initiate image import') self.image_id = imported_image['ImageId'] task_id = imported_image['TaskId'] LOGGER.info('Started image import with image id \'%s\' and task id \'%s\'', self.image_id, task_id) task_status_count = int(get_config_value('ALIBABA_IMAGE_IMPORT_MONITOR_RETRY_COUNT')) task_status_delay = int(get_config_value('ALIBABA_IMAGE_IMPORT_MONITOR_RETRY_DELAY')) if self.monitor_task(task_id, task_status_count, task_status_delay): LOGGER.info('Image \'%s\' imported after %d seconds', self.image_id, time() - start_time) else: canceled_task_msg = 'Image import failed or took too long, ' + \ 'canceling task \'{}\' and '.format(task_id) + \ 'deleting image \'{}\''.format(self.image_id) LOGGER.info(canceled_task_msg) self.client.cancel_task(task_id) self.client.delete_image(self.image_id) raise RuntimeError('Failed to import image \'{}\' after monitoring it for {} retries'. format(self.image_id, task_status_count)) # Add image_id and location (region) to the metadata used for image registration metadata = CloudImageMetadata() metadata.set(self.__class__.__name__, 'image_id', self.image_id) metadata.set(self.__class__.__name__, 'location', get_config_value('ALIBABA_REGION')) # Add tags to image LOGGER.info('Add tags to image \'%s\'', self.image_id) self.client.add_tags(self.image_id, 'image', CloudImageTags(metadata).get()) # Add tags to associated snapshot images_json = self.client.describe_images(self.image_id, None) if not 'Images' in images_json.keys(): LOGGER.error('No image data found for image \'%s\'', self.image_id) LOGGER.error('Unable to tag snapshot.') else: snapshot_id = images_json['Images']['Image'][0] \ ['DiskDeviceMappings']['DiskDeviceMapping'][0]['SnapshotId'] LOGGER.info('Add tags to snapshot \'%s\'', snapshot_id) self.client.add_tags(snapshot_id, 'snapshot', CloudImageTags(metadata).get())
def create_working_dir(self, artifacts_dir): """Create temporary directory""" self.working_dir = tempfile.mkdtemp('', "image_" + self.cloud_type + "_", artifacts_dir) LOGGER.debug("Working directory = '%s'.", self.working_dir)
def read_injected_files(top_call_dir, overall_dest_dir): """ Copy files that need to be injected to a temporary location, which will be accessible during post-install. Two mandatory arguments: a path from where build-image was called a path to initrd directory that will be available during post_install """ # location used by post-install, should be created only if there are files to inject injected_files = 'etc/injected_files' # location used by post-install overall_dest_dir = overall_dest_dir + '/' + injected_files LOGGER.info('Temporary location for injected files: \'%s\'', overall_dest_dir) # include user-specified files files_to_inject = get_list_from_config_yaml('UPDATE_IMAGE_FILES') # add build_info.json prep_build_info_for_injection(files_to_inject) # each injected file directory to be stored in a separate directory "file<number>" count = 0 LOGGER.trace("files_to_inject: %s", files_to_inject) for file in files_to_inject: LOGGER.debug('Injecting file: \'%s\'.', file) src = extract_single_worded_key(file, 'source') dest = extract_single_worded_key(file, 'destination') if 'mode' in file: mode = extract_single_worded_key(file, 'mode') else: mode = None LOGGER.info('Copy \'%s\' to a temporary location for \'%s\'.', src, dest) url = src # treat 'src' as a file path and 'url' as a url if src[0] != '/' and src[0] != '~': # make it an absolute path src = top_call_dir + '/' + src src = abspath(realpath(expanduser(src))) file_holder = overall_dest_dir + '/file' + str(count) + '/' # copy source to "src" # source file name does not need to be preserved; # it will be copied to destination path on BIG-IP source_holder = file_holder + 'src' Path(file_holder).mkdir(parents=True, exist_ok=True) if isfile(src): LOGGER.info('Treating \'%s\' as a file for file injection', src) copy2(src, source_holder) elif isdir(src): LOGGER.info('Treating \'%s\' as a directory for file injection', src) copytree(src, source_holder) else: LOGGER.info('Treating \'%s\' as a URL for the file injection', url) download_file(url, source_holder) # store destination if dest[0] != '/': raise RuntimeError( 'injected file destination \'{}\' must be an absolute path!'. format(dest)) with open(file_holder + 'dest', 'w') as dest_holder: print("{}".format(dest), file=dest_holder) # Store mode. Should be a string consisting of one to four octal digits. if mode: LOGGER.debug('Creating mode holder for mode \'%s\'.', mode) mode_pattern = re.compile('^[0-7][0-7]?[0-7]?[0-7]?$') if not mode_pattern.match(mode): raise RuntimeError('Invalid mode \'' + mode + '\', must be a string ' + 'consisting of one to four octal digits.') with open(file_holder + 'mode', 'w') as mode_holder: print("{}".format(mode), file=mode_holder) count += 1 # end of for loop LOGGER.debug('leaving %s', basename(__file__)) return 0
def read_injected_files(overall_dest_dir): """ Copy file that need to be injected to temporary location, which will be accessible during post-install. One mandatory argument: a path to initrd directory that will be available during post_install """ artifacts_dir = get_config_value("ARTIFACTS_DIR") # location used by post-install, should be created only if there are files to inject injected_files = 'etc/injected_files' # location used by post-install overall_dest_dir = overall_dest_dir + '/' + injected_files LOGGER.info('temporary location for injected files: %s', overall_dest_dir) # include user-specified files files_to_inject = get_list_from_config_yaml('UPDATE_IMAGE_FILES') # include information about installed software on the build machine build_info_file_name = "build_info.json" build_info_source = artifacts_dir + "/" + build_info_file_name build_info_destination = "/" + build_info_file_name files_to_inject.append({ 'source': build_info_source, 'destination': build_info_destination }) build_info = BuildInfo() build_info.to_file(build_info_source) # each injected file directory to be stored in a separate directory "file<number>" count = 0 LOGGER.trace("files_to_inject: %s", files_to_inject) for file in files_to_inject: LOGGER.trace("file: %s", file) src = extract_single_worded_key(file, 'source') dest = extract_single_worded_key(file, 'destination') LOGGER.info('inject %s to temporary location %s', src, dest) file_holder = overall_dest_dir + '/file' + str(count) + '/' # copy source to "src" # source file name does not need to be preserved; # it will be copied to destination path on BIG-IP source_holder = file_holder + 'src' if isfile(src): Path(file_holder).mkdir(parents=True, exist_ok=True) copy2(src, source_holder) elif isdir(src): copytree(src, source_holder) else: raise RuntimeError( '\'{}\' is neither a file nor a directory, cannot inject it!'. format(src)) # store destination if dest[0] != '/': raise RuntimeError( 'injected file destination \'{}\' must be an absolute path!'. format(dest)) with open(file_holder + 'dest', 'w') as dest_holder: print("{}".format(dest), file=dest_holder) count += 1 # end of for loop LOGGER.debug('leaving %s', basename(__file__)) return 0
def read_injected_files(top_call_dir, overall_dest_dir): """ Copy file that need to be injected to temporary location, which will be accessible during post-install. Two mandatory arguments: a path from where build-image was called a path to initrd directory that will be available during post_install """ # location used by post-install, should be created only if there are files to inject injected_files = 'etc/injected_files' # location used by post-install overall_dest_dir = overall_dest_dir + '/' + injected_files LOGGER.info('temporary location for injected files: %s', overall_dest_dir) # include user-specified files files_to_inject = get_list_from_config_yaml('UPDATE_IMAGE_FILES') # add build_info.json prep_build_info_for_injection(files_to_inject) # each injected file directory to be stored in a separate directory "file<number>" count = 0 LOGGER.trace("files_to_inject: %s", files_to_inject) for file in files_to_inject: LOGGER.trace("file: %s", file) src = extract_single_worded_key(file, 'source') if src[0] != '/' and src[0] != '~': # make it an absolute path src = top_call_dir + '/' + src src = abspath(realpath(expanduser(src))) dest = extract_single_worded_key(file, 'destination') LOGGER.info('inject %s to temporary location %s', src, dest) file_holder = overall_dest_dir + '/file' + str(count) + '/' # copy source to "src" # source file name does not need to be preserved; # it will be copied to destination path on BIG-IP source_holder = file_holder + 'src' if isfile(src): Path(file_holder).mkdir(parents=True, exist_ok=True) copy2(src, source_holder) elif isdir(src): copytree(src, source_holder) else: raise RuntimeError( '\'{}\' is neither a file nor a directory, cannot inject it!'. format(src)) # store destination if dest[0] != '/': raise RuntimeError( 'injected file destination \'{}\' must be an absolute path!'. format(dest)) with open(file_holder + 'dest', 'w') as dest_holder: print("{}".format(dest), file=dest_holder) count += 1 # end of for loop LOGGER.debug('leaving %s', basename(__file__)) return 0
def extract(self): """Extract the qcow2 disk out of zip.""" LOGGER.debug("Extracting '.qcow2 disk file from [%s].", self.input_disk_path) self.disk_to_upload = BaseDisk.decompress(self.input_disk_path, '.qcow2', self.working_dir) LOGGER.info("Alibaba disk_to_upload = '%s'", self.disk_to_upload)