def clean_up(self):
"""Clean-up."""
if self.disk_to_upload is not None and os.path.exists(
self.disk_to_upload):
LOGGER.debug(
"Deleting the local disk that was (or had to be) uploaded:= '%s'",
self.disk_to_upload)
os.remove(self.disk_to_upload)
else:
failed_delete_file_msg = "Could not find (and delete) the local disk that " + \
"was (or had to be) uploaded: {}".format(self.disk_to_upload)
LOGGER.debug(failed_delete_file_msg)
LOGGER.debug("Cleaning up the uploaded disk from Alibaba storage")
self.upload_cleanup()
def main():
"""main publish telemetry information function"""
# create log handler for the global LOGGER
create_log_handler()
# gather telemetry info
build_info_telemetry = BuildInfoTelemetry()
LOGGER.debug("telemetry info:")
LOGGER.debug(build_info_telemetry.build_info)
version = build_info_telemetry.build_info['product']['version']
# Check if specific api key is set, if not use default
if environ.get("F5_TEEM_API_KEY") is not None:
environ['F5_TEEM_API_ENVIRONMENT'] = "staging"
f5_api_key = environ.get("F5_TEEM_API_KEY")
else:
f5_api_key = 'mmhJU2sCd63BznXAXDh4kxLIyfIMm3Ar'
generated_uuid = str(uuid.uuid4())
LOGGER.debug("telemetry UUID: %s", generated_uuid)
client_info = {
'name': 'f5-image-generator',
'version': str(version),
'id': generated_uuid
}
telemetry_client = AnonymousDeviceClient(client_info, api_key=f5_api_key)
retrier = Retrier(_publish_telemetry_database, build_info_telemetry,
telemetry_client)
retrier.tries = int(get_config_value('PUBLISH_TELEMETRY_TASK_RETRY_COUNT'))
retrier.delay = int(get_config_value('PUBLISH_TELEMETRY_TASK_RETRY_DELAY'))
if retrier.execute():
LOGGER.info("Publishing to telemetry success.")
return True
LOGGER.info("Publishing to telemetry did not succeed.")
sys.exit(0)
def get_python_modules():
"""Collect version information for active Python modules"""
LOGGER.debug("Collecting version information for active Python modules")
python_command = BuildInfo.package_manager_commands["python"]
python_modules = _command_key_values_to_dict(*python_command)
return python_modules
def create_image(self, image_name):
""" Create image implementation for Alibaba """
images_json = self.client.describe_images(None, image_name)
if int(images_json['TotalCount']) == 0:
LOGGER.debug('No old images named \'%s\' were found', image_name)
else:
# image names are unique, delete only one image
image_id = images_json['Images']['Image'][0]['ImageId']
LOGGER.info('Image \'%s\' already exists, its id is \'%s\', deleting it', image_name,
image_id)
self.client.delete_image(image_id)
# start image creation
LOGGER.info('Started creation of image \'%s\' at %s', image_name,
datetime.datetime.now().strftime('%H:%M:%S'))
start_time = time()
imported_image = self.client.import_image(get_config_value('ALIBABA_BUCKET'),
self.disk.uploaded_disk_name, image_name)
if 'Code' in imported_image.keys():
if imported_image['Code'] == 'InvalidOSSObject.NotFound':
raise RuntimeError('ImportImageRequest could not find uloaded disk \'' +
image_name + '\'')
if imported_image['Code'] == 'InvalidImageName.Duplicated':
raise RuntimeError('Image \'' + image_name + '\' still exists, ' +
'should have been removed by this point')
if imported_image['Code'] == 'ImageIsImporting':
raise RuntimeError('Another image named \'' + image_name + '\' is in the ' +
'process of importing, probably from the previous run. ' +
'Delete it first.')
if 'ImageId' not in imported_image.keys() or 'TaskId' not in imported_image.keys():
LOGGER.info('Alibaba response to ImportImageRequest:')
LOGGER.info(json.dumps(imported_image, sort_keys=True, indent=4,
separators=(',', ': ')))
raise RuntimeError('ImageId and/or TaskId were not found in the response ' +
'cannot initiate image import')
self.image_id = imported_image['ImageId']
task_id = imported_image['TaskId']
LOGGER.info('Started image import with image id \'%s\' and task id \'%s\'', self.image_id,
task_id)
task_status_count = int(get_config_value('ALIBABA_IMAGE_IMPORT_MONITOR_RETRY_COUNT'))
task_status_delay = int(get_config_value('ALIBABA_IMAGE_IMPORT_MONITOR_RETRY_DELAY'))
if self.monitor_task(task_id, task_status_count, task_status_delay):
LOGGER.info('Image \'%s\' imported after %d seconds',
self.image_id, time() - start_time)
else:
canceled_task_msg = 'Image import failed or took too long, ' + \
'canceling task \'{}\' and '.format(task_id) + \
'deleting image \'{}\''.format(self.image_id)
LOGGER.info(canceled_task_msg)
self.client.cancel_task(task_id)
self.client.delete_image(self.image_id)
raise RuntimeError('Failed to import image \'{}\' after monitoring it for {} retries'.
format(self.image_id, task_status_count))
# Add image_id and location (region) to the metadata used for image registration
metadata = CloudImageMetadata()
metadata.set(self.__class__.__name__, 'image_id', self.image_id)
metadata.set(self.__class__.__name__, 'location', get_config_value('ALIBABA_REGION'))
# Add tags to image
LOGGER.info('Add tags to image \'%s\'', self.image_id)
self.client.add_tags(self.image_id, 'image', CloudImageTags(metadata).get())
# Add tags to associated snapshot
images_json = self.client.describe_images(self.image_id, None)
if not 'Images' in images_json.keys():
LOGGER.error('No image data found for image \'%s\'', self.image_id)
LOGGER.error('Unable to tag snapshot.')
else:
snapshot_id = images_json['Images']['Image'][0] \
['DiskDeviceMappings']['DiskDeviceMapping'][0]['SnapshotId']
LOGGER.info('Add tags to snapshot \'%s\'', snapshot_id)
self.client.add_tags(snapshot_id, 'snapshot', CloudImageTags(metadata).get())
def create_working_dir(self, artifacts_dir):
"""Create temporary directory"""
self.working_dir = tempfile.mkdtemp('',
"image_" + self.cloud_type + "_",
artifacts_dir)
LOGGER.debug("Working directory = '%s'.", self.working_dir)
def read_injected_files(top_call_dir, overall_dest_dir):
"""
Copy files that need to be injected to a temporary location,
which will be accessible during post-install.
Two mandatory arguments:
a path from where build-image was called
a path to initrd directory that will be available during post_install
"""
# location used by post-install, should be created only if there are files to inject
injected_files = 'etc/injected_files' # location used by post-install
overall_dest_dir = overall_dest_dir + '/' + injected_files
LOGGER.info('Temporary location for injected files: \'%s\'',
overall_dest_dir)
# include user-specified files
files_to_inject = get_list_from_config_yaml('UPDATE_IMAGE_FILES')
# add build_info.json
prep_build_info_for_injection(files_to_inject)
# each injected file directory to be stored in a separate directory "file<number>"
count = 0
LOGGER.trace("files_to_inject: %s", files_to_inject)
for file in files_to_inject:
LOGGER.debug('Injecting file: \'%s\'.', file)
src = extract_single_worded_key(file, 'source')
dest = extract_single_worded_key(file, 'destination')
if 'mode' in file:
mode = extract_single_worded_key(file, 'mode')
else:
mode = None
LOGGER.info('Copy \'%s\' to a temporary location for \'%s\'.', src,
dest)
url = src # treat 'src' as a file path and 'url' as a url
if src[0] != '/' and src[0] != '~':
# make it an absolute path
src = top_call_dir + '/' + src
src = abspath(realpath(expanduser(src)))
file_holder = overall_dest_dir + '/file' + str(count) + '/'
# copy source to "src"
# source file name does not need to be preserved;
# it will be copied to destination path on BIG-IP
source_holder = file_holder + 'src'
Path(file_holder).mkdir(parents=True, exist_ok=True)
if isfile(src):
LOGGER.info('Treating \'%s\' as a file for file injection', src)
copy2(src, source_holder)
elif isdir(src):
LOGGER.info('Treating \'%s\' as a directory for file injection',
src)
copytree(src, source_holder)
else:
LOGGER.info('Treating \'%s\' as a URL for the file injection', url)
download_file(url, source_holder)
# store destination
if dest[0] != '/':
raise RuntimeError(
'injected file destination \'{}\' must be an absolute path!'.
format(dest))
with open(file_holder + 'dest', 'w') as dest_holder:
print("{}".format(dest), file=dest_holder)
# Store mode. Should be a string consisting of one to four octal digits.
if mode:
LOGGER.debug('Creating mode holder for mode \'%s\'.', mode)
mode_pattern = re.compile('^[0-7][0-7]?[0-7]?[0-7]?$')
if not mode_pattern.match(mode):
raise RuntimeError('Invalid mode \'' + mode +
'\', must be a string ' +
'consisting of one to four octal digits.')
with open(file_holder + 'mode', 'w') as mode_holder:
print("{}".format(mode), file=mode_holder)
count += 1
# end of for loop
LOGGER.debug('leaving %s', basename(__file__))
return 0
def read_injected_files(overall_dest_dir):
"""
Copy file that need to be injected to temporary location,
which will be accessible during post-install.
One mandatory argument: a path to initrd directory that will be available during post_install
"""
artifacts_dir = get_config_value("ARTIFACTS_DIR")
# location used by post-install, should be created only if there are files to inject
injected_files = 'etc/injected_files' # location used by post-install
overall_dest_dir = overall_dest_dir + '/' + injected_files
LOGGER.info('temporary location for injected files: %s', overall_dest_dir)
# include user-specified files
files_to_inject = get_list_from_config_yaml('UPDATE_IMAGE_FILES')
# include information about installed software on the build machine
build_info_file_name = "build_info.json"
build_info_source = artifacts_dir + "/" + build_info_file_name
build_info_destination = "/" + build_info_file_name
files_to_inject.append({
'source': build_info_source,
'destination': build_info_destination
})
build_info = BuildInfo()
build_info.to_file(build_info_source)
# each injected file directory to be stored in a separate directory "file<number>"
count = 0
LOGGER.trace("files_to_inject: %s", files_to_inject)
for file in files_to_inject:
LOGGER.trace("file: %s", file)
src = extract_single_worded_key(file, 'source')
dest = extract_single_worded_key(file, 'destination')
LOGGER.info('inject %s to temporary location %s', src, dest)
file_holder = overall_dest_dir + '/file' + str(count) + '/'
# copy source to "src"
# source file name does not need to be preserved;
# it will be copied to destination path on BIG-IP
source_holder = file_holder + 'src'
if isfile(src):
Path(file_holder).mkdir(parents=True, exist_ok=True)
copy2(src, source_holder)
elif isdir(src):
copytree(src, source_holder)
else:
raise RuntimeError(
'\'{}\' is neither a file nor a directory, cannot inject it!'.
format(src))
# store destination
if dest[0] != '/':
raise RuntimeError(
'injected file destination \'{}\' must be an absolute path!'.
format(dest))
with open(file_holder + 'dest', 'w') as dest_holder:
print("{}".format(dest), file=dest_holder)
count += 1
# end of for loop
LOGGER.debug('leaving %s', basename(__file__))
return 0
def read_injected_files(top_call_dir, overall_dest_dir):
"""
Copy file that need to be injected to temporary location,
which will be accessible during post-install.
Two mandatory arguments:
a path from where build-image was called
a path to initrd directory that will be available during post_install
"""
# location used by post-install, should be created only if there are files to inject
injected_files = 'etc/injected_files' # location used by post-install
overall_dest_dir = overall_dest_dir + '/' + injected_files
LOGGER.info('temporary location for injected files: %s', overall_dest_dir)
# include user-specified files
files_to_inject = get_list_from_config_yaml('UPDATE_IMAGE_FILES')
# add build_info.json
prep_build_info_for_injection(files_to_inject)
# each injected file directory to be stored in a separate directory "file<number>"
count = 0
LOGGER.trace("files_to_inject: %s", files_to_inject)
for file in files_to_inject:
LOGGER.trace("file: %s", file)
src = extract_single_worded_key(file, 'source')
if src[0] != '/' and src[0] != '~':
# make it an absolute path
src = top_call_dir + '/' + src
src = abspath(realpath(expanduser(src)))
dest = extract_single_worded_key(file, 'destination')
LOGGER.info('inject %s to temporary location %s', src, dest)
file_holder = overall_dest_dir + '/file' + str(count) + '/'
# copy source to "src"
# source file name does not need to be preserved;
# it will be copied to destination path on BIG-IP
source_holder = file_holder + 'src'
if isfile(src):
Path(file_holder).mkdir(parents=True, exist_ok=True)
copy2(src, source_holder)
elif isdir(src):
copytree(src, source_holder)
else:
raise RuntimeError(
'\'{}\' is neither a file nor a directory, cannot inject it!'.
format(src))
# store destination
if dest[0] != '/':
raise RuntimeError(
'injected file destination \'{}\' must be an absolute path!'.
format(dest))
with open(file_holder + 'dest', 'w') as dest_holder:
print("{}".format(dest), file=dest_holder)
count += 1
# end of for loop
LOGGER.debug('leaving %s', basename(__file__))
return 0
def extract(self):
"""Extract the qcow2 disk out of zip."""
LOGGER.debug("Extracting '.qcow2 disk file from [%s].", self.input_disk_path)
self.disk_to_upload = BaseDisk.decompress(self.input_disk_path, '.qcow2', self.working_dir)
LOGGER.info("Alibaba disk_to_upload = '%s'", self.disk_to_upload)
