def read_update_job_info(file_name):
update_jobs = []
if not os.path.isfile(file_name):
print(
'*** Warning - Unable to read existing update jobs as given file does not exist: {0}'
.format(file_name))
return update_jobs
print('')
print('')
print('Parsing: {0}'.format(file_name))
parser = ConfigParser.ConfigParser()
parser.readfp(open(file_name))
# parse
for section in parser.sections():
if section.startswith('UpdateJob.'):
updates_xml = bldinstallercommon.safe_config_key_fetch(
parser, section, 'updates_xml')
repo_platform = bldinstallercommon.safe_config_key_fetch(
parser, section, 'repo_platform')
repo_specifier = bldinstallercommon.safe_config_key_fetch(
parser, section, 'repo_specifier')
repo_update_state = bldinstallercommon.safe_config_key_fetch(
parser, section, 'repo_update_state')
update_job = UpdateJob(section, updates_xml, repo_platform,
repo_specifier, repo_update_state)
update_jobs.append(update_job)
# validate
for item in update_jobs:
item.validate()
return update_jobs
def __init__(self, target_config, server_base_url_override, configurations_root_dir, key_substitution_list):
"""Init data based on the target configuration"""
self.server_list = []
self.pkg_templates_dir_list = []
self.default_server = None
self.configurations_root_dir = configurations_root_dir
self.key_substitution_list = key_substitution_list
# get packages tempalates src dir first
pkg_templates_dir = os.path.normpath(bldinstallercommon.config_section_map(target_config,'PackageTemplates')['template_dirs'])
self.pkg_templates_dir_list = pkg_templates_dir.replace(' ', '').rstrip(',\n').split(',')
# next read server list
if server_base_url_override:
server_obj = ArchiveLocationResolver.ArchiveRemoteLocation('default_server_name', server_base_url_override, '')
self.server_list.append(server_obj)
else:
for section in target_config.sections():
if section.startswith(SERVER_NAMESPACE):
server_name = section.split('.')[-1]
base_url = bldinstallercommon.safe_config_key_fetch(target_config, section, 'base_url')
base_path = bldinstallercommon.safe_config_key_fetch(target_config, section, 'base_path')
base_path.replace(' ', '')
# if base path is defined, then the following logic applies:
# if script is used in testclient mode fetch the packages from "RnD" location
# otherwise fetch packages from "release" location.
# If the base_path is not defined, use the address as-is
if base_path:
base_path = base_path + PACKAGE_REMOTE_LOCATION_RELEASE
server_obj = ArchiveLocationResolver.ArchiveRemoteLocation(server_name, base_url, base_path)
self.server_list.append(server_obj)
if len(self.server_list) == 1:
self.default_server = self.server_list[0]
def handle_installer_build(optionDict, installer_type, branch, arch):
conf_file = optionDict['RELEASE_DESCRIPTION_FILE']
print('Parsing [{0}] installer build jobs from: {0}'.format(conf_file))
if not os.path.isfile(conf_file):
raise IOError('*** Fatal error! Given file does not exist: {0}'.format(
conf_file))
init_env(optionDict)
license_type = optionDict['LICENSE']
platform = optionDict['HOST_PLATFORM']
packages_base_url = optionDict['PACKAGE_STORAGE_SERVER_PATH_HTTP']
# parse conf file
parser = ConfigParser.ConfigParser()
parser.readfp(open(conf_file))
section_name = branch + '.' + 'global'
global_version = bldinstallercommon.safe_config_key_fetch(
parser, section_name, 'version')
global_version_tag = bldinstallercommon.safe_config_key_fetch(
parser, section_name, 'version_tag')
if not global_version:
raise RuntimeError(
'*** Fatal error! Invalid values in {0} -> {1}'.format(
conf_file, section_name))
# parse build jobs
job_list = get_job_list(optionDict, installer_type, branch, arch,
global_version, global_version_tag)
if (len(job_list) == 0):
raise RuntimeError(
'*** Fatal error! No [{0}] installer build jobs found from: {1}. Probably an error?'
.format(installer_type, conf_file))
installer_output_dir = os.path.join(
SCRIPT_ROOT_DIR, pkg_constants.INSTALLER_OUTPUT_DIR_NAME)
rta_descr_output_dir = os.path.join(
SCRIPT_ROOT_DIR, pkg_constants.RTA_DESCRIPTION_FILE_DIR_NAME)
bldinstallercommon.create_dirs(rta_descr_output_dir)
# create rta description file
architecture = bldinstallercommon.get_architecture()
plat_suffix = bldinstallercommon.get_platform_suffix()
rta_description_file_name = os.path.join(
rta_descr_output_dir, pkg_constants.RTA_DESCRIPTION_FILE_NAME_BASE +
'-' + plat_suffix + '-' + architecture + '.txt')
# handle build jobs
for job in job_list:
# create installer
type_arg = '--online' if 'online' in installer_type.lower(
) else '--offline'
creation_ok = create_installer(job, packages_base_url, type_arg)
# write the rta description file only if installer creation was ok
if (creation_ok):
rta_description_file = open(rta_description_file_name, 'a')
rta_description_file.write(job.installer_name + ' ' +
job.rta_key_list + '\n')
rta_description_file.close()
# if "/installer_output" directory is empty -> error
if not os.listdir(installer_output_dir):
raise RuntimeError(
'*** Fatal error! No installers generated into: %s' %
installer_output_dir)
def handle_repo_build(optionDict, branch, arch, update_staging_repo, update_production_repo):
conf_file = optionDict['RELEASE_DESCRIPTION_FILE']
if not os.path.isfile(conf_file):
raise IOError('*** Fatal error! Given file does not exist: %s' % conf_file)
init_env(optionDict)
packages_base_url = optionDict['PACKAGE_STORAGE_SERVER_PATH_HTTP']
ifw_base_url = optionDict['IFW_TOOLS_BASE_URL']
release_tools_dir = SCRIPT_ROOT_DIR
# parse conf file
parser = ConfigParser.ConfigParser()
parser.readfp(open(conf_file))
section_name = branch + '.' + 'global'
global_version = bldinstallercommon.safe_config_key_fetch(parser, section_name, 'version')
global_version_tag = bldinstallercommon.safe_config_key_fetch(parser, section_name, 'version_tag')
# parse build jobs
repo_job_list = get_repo_job_list(optionDict, branch, arch, global_version, global_version_tag)
if (len(repo_job_list) == 0):
raise RuntimeError('*** Fatal error! No repository build jobs found. Probably an error? %s' % conf_file)
# init repo dirs
init_repositories(optionDict, repo_job_list)
# is this snapshot build? Then enable component version number forced update
forced_version_number_bump = False
if update_staging_repo and not update_production_repo:
forced_version_number_bump = True
if optionDict.get('FORCE_VERSION_NUMBER_INCREASE') in ['yes', 'true', '1']:
forced_version_number_bump = True
# create rta description file
rta_descr_output_dir = os.path.join(SCRIPT_ROOT_DIR, pkg_constants.RTA_DESCRIPTION_FILE_DIR_NAME)
bldinstallercommon.create_dirs(rta_descr_output_dir)
architecture = bldinstallercommon.get_architecture()
plat_suffix = bldinstallercommon.get_platform_suffix()
rta_description_file_name = os.path.join( rta_descr_output_dir, pkg_constants.RTA_DESCRIPTION_FILE_NAME_BASE + '-' + plat_suffix + '-' + architecture + '-repo.txt')
# handle repo build jobs
for job in repo_job_list:
create_online_repository(job, packages_base_url, forced_version_number_bump)
# determine testination path on test server
dest_path_repository, dest_path_pkg = generate_repo_dest_path_pending(optionDict, job)
# copy repo content to test server
source_path_repository = os.path.join(release_tools_dir, 'repository')
source_path_pkg = os.path.join(release_tools_dir, 'pkg')
push_online_repository(optionDict, optionDict['PKG_STAGING_SERVER'], optionDict['PKG_STAGING_SERVER_UNAME'], source_path_repository, dest_path_repository)
push_online_repository(optionDict, optionDict['PKG_STAGING_SERVER'], optionDict['PKG_STAGING_SERVER_UNAME'], source_path_pkg, dest_path_pkg)
# remove local repository and pkg directories
bldinstallercommon.remove_tree(source_path_repository)
bldinstallercommon.remove_tree(source_path_pkg)
# update repo in testing area
staging_repo_updated, dummy = update_online_repo(optionDict, job, update_staging_repo, update_production_repo)
# write the rta description file only if staging repository creation was ok
if (staging_repo_updated):
rta_description_file = open(rta_description_file_name, 'a')
rta_description_file.write(job.repo_url_specifier + ' ' + job.rta_key_list + '\n')
rta_description_file.close()
def handle_installer_build(optionDict, installer_type, branch, arch):
conf_file = optionDict["RELEASE_DESCRIPTION_FILE"]
print("Parsing [{0}] installer build jobs from: {0}".format(conf_file))
if not os.path.isfile(conf_file):
raise IOError("*** Fatal error! Given file does not exist: {0}".format(conf_file))
init_env(optionDict)
license_type = optionDict["LICENSE"]
platform = optionDict["HOST_PLATFORM"]
packages_base_url = optionDict["PACKAGE_STORAGE_SERVER_PATH_HTTP"]
# parse conf file
parser = ConfigParser.ConfigParser()
parser.readfp(open(conf_file))
section_name = branch + "." + "global"
global_version = bldinstallercommon.safe_config_key_fetch(parser, section_name, "version")
global_version_tag = bldinstallercommon.safe_config_key_fetch(parser, section_name, "version_tag")
if not global_version:
raise RuntimeError("*** Fatal error! Invalid values in {0} -> {1}".format(conf_file, section_name))
# parse build jobs
job_list = get_job_list(optionDict, installer_type, branch, arch, global_version, global_version_tag)
if len(job_list) == 0:
raise RuntimeError(
"*** Fatal error! No [{0}] installer build jobs found from: {1}. Probably an error?".format(
installer_type, conf_file
)
)
installer_output_dir = os.path.join(SCRIPT_ROOT_DIR, pkg_constants.INSTALLER_OUTPUT_DIR_NAME)
rta_descr_output_dir = os.path.join(SCRIPT_ROOT_DIR, pkg_constants.RTA_DESCRIPTION_FILE_DIR_NAME)
bldinstallercommon.create_dirs(rta_descr_output_dir)
# create rta description file
architecture = bldinstallercommon.get_architecture()
plat_suffix = bldinstallercommon.get_platform_suffix()
rta_description_file_name = os.path.join(
rta_descr_output_dir,
pkg_constants.RTA_DESCRIPTION_FILE_NAME_BASE + "-" + plat_suffix + "-" + architecture + ".txt",
)
# handle build jobs
for job in job_list:
# create installer
type_arg = "--online" if "online" in installer_type.lower() else "--offline"
creation_ok = create_installer(job, packages_base_url, type_arg)
# write the rta description file only if installer creation was ok
if creation_ok:
rta_description_file = open(rta_description_file_name, "a")
rta_description_file.write(job.installer_name + " " + job.rta_key_list + "\n")
rta_description_file.close()
# if "/installer_output" directory is empty -> error
if not os.listdir(installer_output_dir):
raise RuntimeError("*** Fatal error! No installers generated into: %s" % installer_output_dir)
def __init__(self, archive, package_name, parent_target_install_base,
archive_server_name, target_config,
archive_location_resolver, key_value_substitution_list):
self.archive_uri = bldinstallercommon.config_section_map(
target_config, archive)['archive_uri']
self.archive_action = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'archive_action')
self.extract_archive = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'extract_archive')
self.package_strip_dirs = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'package_strip_dirs')
self.package_finalize_items = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'package_finalize_items')
# parent's 'target_install_base'
self.parent_target_install_base = parent_target_install_base
# in case the individual archive needs to be installed outside the root dir specified by the parent component
self.target_install_base = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'target_install_base')
# this is relative to 1) current archive's 'target_install_base' 2) parent components 'target_install_base'. (1) takes priority
self.target_install_dir = bldinstallercommon.safe_config_key_fetch(
target_config, archive,
'target_install_dir').lstrip(os.path.sep)
self.rpath_target = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'rpath_target')
self.component_sha1_file = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'component_sha1_file')
self.nomalize_archive_uri(package_name, archive_server_name,
archive_location_resolver)
self.archive_name = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'archive_name')
if not self.archive_name:
self.archive_name = self.path_leaf(self.archive_uri)
# Parse unnecessary extensions away from filename (QTBUG-39219)
known_archive_types = [
'.tar.gz', '.tar', '.zip', '.tar.xz', '.tar.bz2'
]
for item in known_archive_types:
if self.archive_name.endswith(item):
self.archive_name = self.archive_name.replace(item, '')
if not self.archive_name.endswith('.7z'):
self.archive_name += '.7z'
# substitute key-value pairs if any
for item in key_value_substitution_list:
self.target_install_base = self.target_install_base.replace(
item[0], item[1])
self.target_install_dir = self.target_install_dir.replace(
item[0], item[1])
self.archive_name = self.archive_name.replace(item[0], item[1])
def __init__(self, target_config, server_base_url_override,
configurations_root_dir, key_substitution_list):
"""Init data based on the target configuration"""
self.server_list = []
self.pkg_templates_dir_list = []
self.default_server = None
self.configurations_root_dir = configurations_root_dir
self.key_substitution_list = key_substitution_list
# get packages tempalates src dir first
pkg_templates_dir = os.path.normpath(
bldinstallercommon.config_section_map(
target_config, 'PackageTemplates')['template_dirs'])
self.pkg_templates_dir_list = pkg_templates_dir.replace(
' ', '').rstrip(',\n').split(',')
# next read server list
if server_base_url_override:
server_obj = ArchiveLocationResolver.ArchiveRemoteLocation(
'default_server_name', server_base_url_override, '')
self.server_list.append(server_obj)
else:
for section in target_config.sections():
if section.startswith(SERVER_NAMESPACE):
server_name = section.split('.')[-1]
base_url = bldinstallercommon.safe_config_key_fetch(
target_config, section, 'base_url')
base_path = bldinstallercommon.safe_config_key_fetch(
target_config, section, 'base_path')
base_path.replace(' ', '')
# if base path is defined, then the following logic applies:
# if script is used in testclient mode fetch the packages from "RnD" location
# otherwise fetch packages from "release" location.
# If the base_path is not defined, use the address as-is
if base_path:
base_path = base_path + PACKAGE_REMOTE_LOCATION_RELEASE
server_obj = ArchiveLocationResolver.ArchiveRemoteLocation(
server_name, base_url, base_path)
self.server_list.append(server_obj)
if len(self.server_list) == 1:
self.default_server = self.server_list[0]
def read_update_job_info(file_name):
update_jobs = []
if not os.path.isfile(file_name):
print('*** Warning - Unable to read existing update jobs as given file does not exist: {0}'.format(file_name))
return update_jobs
print('')
print('')
print('Parsing: {0}'.format(file_name))
parser = ConfigParser.ConfigParser()
parser.readfp(open(file_name))
# parse
for section in parser.sections():
if section.startswith('UpdateJob.'):
updates_xml = bldinstallercommon.safe_config_key_fetch(parser, section, 'updates_xml')
repo_platform = bldinstallercommon.safe_config_key_fetch(parser, section, 'repo_platform')
repo_specifier = bldinstallercommon.safe_config_key_fetch(parser, section, 'repo_specifier')
repo_update_state = bldinstallercommon.safe_config_key_fetch(parser, section, 'repo_update_state')
update_job = UpdateJob(section, updates_xml, repo_platform, repo_specifier, repo_update_state)
update_jobs.append(update_job)
# validate
for item in update_jobs:
item.validate()
return update_jobs
def __init__(self, archive, package_name, archive_server_name, target_config, archive_location_resolver, key_value_substitution_list):
self.archive_uri = bldinstallercommon.config_section_map(target_config, archive)['archive_uri']
self.extract_archive = bldinstallercommon.safe_config_key_fetch(target_config, archive, 'extract_archive')
self.package_strip_dirs = bldinstallercommon.safe_config_key_fetch(target_config, archive, 'package_strip_dirs')
self.package_finalize_items = bldinstallercommon.safe_config_key_fetch(target_config, archive, 'package_finalize_items')
self.target_install_dir = bldinstallercommon.safe_config_key_fetch(target_config, archive, 'target_install_dir') # todo, is needed?
self.rpath_target = bldinstallercommon.safe_config_key_fetch(target_config, archive, 'rpath_target')
self.component_sha1_file = bldinstallercommon.safe_config_key_fetch(target_config, archive, 'component_sha1_file')
self.nomalize_archive_uri(package_name, archive_server_name, archive_location_resolver)
self.archive_name = bldinstallercommon.safe_config_key_fetch(target_config, archive, 'archive_name')
if not self.archive_name:
self.archive_name = self.path_leaf(self.archive_uri)
# Parse unnecessary extensions away from filename (QTBUG-39219)
known_archive_types = ['.tar.gz', '.tar', '.zip', '.tar.xz', '.tar.bz2']
for item in known_archive_types:
if self.archive_name.endswith(item):
self.archive_name = self.archive_name.replace(item, '')
if not self.archive_name.endswith('.7z'):
self.archive_name += '.7z'
# substitute key-value pairs if any
for item in key_value_substitution_list:
self.target_install_dir = self.target_install_dir.replace(item[0], item[1])
self.archive_name = self.archive_name.replace(item[0], item[1])
def __init__(self, archive, package_name, archive_server_name,
target_config, archive_location_resolver,
key_value_substitution_list):
self.archive_uri = bldinstallercommon.config_section_map(
target_config, archive)['archive_uri']
self.extract_archive = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'extract_archive')
self.package_strip_dirs = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'package_strip_dirs')
self.package_finalize_items = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'package_finalize_items')
self.target_install_dir = bldinstallercommon.safe_config_key_fetch(
target_config, archive,
'target_install_dir') # todo, is needed?
self.rpath_target = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'rpath_target')
self.nomalize_archive_uri(package_name, archive_server_name,
archive_location_resolver)
self.archive_name = bldinstallercommon.safe_config_key_fetch(
target_config, archive, 'archive_name')
if not self.archive_name:
self.archive_name = self.path_leaf(self.archive_uri)
# Parse unnecessary extensions away from filename (QTBUG-39219)
known_archive_types = [
'.tar.gz', '.tar', '.zip', '.tar.xz', '.tar.bz2'
]
for item in known_archive_types:
if self.archive_name.endswith(item):
self.archive_name = self.archive_name.replace(item, '')
if not self.archive_name.endswith('.7z'):
self.archive_name += '.7z'
# substitute key-value pairs if any
for item in key_value_substitution_list:
self.target_install_dir = self.target_install_dir.replace(
item[0], item[1])
self.archive_name = self.archive_name.replace(item[0], item[1])
def get_job_list(optionDict, job_type_specifier, branch, arch, global_version, global_version_tag):
conf_file = optionDict['RELEASE_DESCRIPTION_FILE']
license_type = optionDict['LICENSE']
platform = optionDict['HOST_PLATFORM']
conf_file_base_dir = optionDict['CONFIGURATIONS_FILE_BASE_DIR']
ifw_base_url = optionDict['IFW_TOOLS_BASE_URL']
print('Get [{0}] build job list for: {1}'.format(job_type_specifier, platform + '-' + arch))
if not os.path.isfile(conf_file):
raise IOError('*** Fatal error! Given file does not exist: %s' % conf_file)
# ensure the string ends with '/'
if not ifw_base_url.endswith('/'):
ifw_base_url += '/'
parser = ConfigParser.ConfigParser()
parser.readfp(open(conf_file))
# validate job type
if not is_valid_job_type(job_type_specifier):
raise RuntimeError('*** Fatal error! Unsupported job type specifier given: %s' % job_type_specifier)
# determine which ifw tools to use (which platform)
ifw_tools = bldinstallercommon.safe_config_key_fetch(parser, 'ifwtools', platform + '-' + arch)
if not ifw_tools:
raise RuntimeError('*** Fatal error! Unable to find ifw tools for_ %s' % (platform + '-' + arch))
ifw_tools_url = urlparse.urljoin(ifw_base_url, ifw_tools)
# check if repository build job
is_repo_job = False
if job_type_specifier == 'repository':
is_repo_job = True
# first read global arg substitution list applicable for all build jobs in this file
global_arg_substitution_list = bldinstallercommon.safe_config_key_fetch(parser, 'release.global', 'arg_substitution_list')
# parse
job_list = []
for s in parser.sections():
section_parts = s.split('.')
if (len(section_parts) < 5):
continue
if (section_parts[0] == branch and section_parts[2] == job_type_specifier):
# Check first if this job was assigned to dedicated machine label
machine_label = bldinstallercommon.safe_config_key_fetch(parser, s, 'assign_to_machine_label')
if machine_label:
if not machine_label == optionDict['TARGET_ENV']:
continue
else: # If not then check against the platform and arch
if not ((section_parts[3] == platform) and (section_parts[4] == arch)):
continue
# parse from conf file
version_number = bldinstallercommon.safe_config_key_fetch(parser, s, 'version_number')
if not version_number:
version_number = global_version
version_number_tag = bldinstallercommon.safe_config_key_fetch(parser, s, 'version_number_tag')
if not version_number_tag:
version_number_tag = global_version_tag
arg_configurations_file = bldinstallercommon.safe_config_key_fetch(parser, s, 'arg_configurations_file')
if not arg_configurations_file:
raise RuntimeError('*** Fatal error! Configuration file not defined for: %s' % s)
# for triggering rta later on if specified
rta_key_list = bldinstallercommon.safe_config_key_fetch(parser, s, 'rta_key_list')
# preferred installer name
installer_name = bldinstallercommon.safe_config_key_fetch(parser, s, 'installer_name')
arg_substitution_list = bldinstallercommon.safe_config_key_fetch(parser, s, 'arg_substitution_list')
arg_substitution_list += ',' + global_arg_substitution_list
arg_substitution_list += ',' + "%BUILD_NUMBER%=" + optionDict['BUILD_NUMBER']
arg_substitution_list = arg_substitution_list.replace('\n', '')
arg_substitution_list = preformat_global_version_and_tag(arg_substitution_list, global_version, global_version_tag)
repo_content_type = ''
repo_components_to_update = ''
repo_url_specifier = ''
# if online repo job
if job_type_specifier == 'repository':
repo_content_type = bldinstallercommon.safe_config_key_fetch(parser, s, 'repo_content_type')
if not repo_content_type:
repo_content_type = arg_configurations_file.split("/")[-1] # if the 'repo_content_type' (for temp dir name) is not defined then parse it from the conf file
repo_components_to_update = bldinstallercommon.safe_config_key_fetch(parser, s, 'repo_components_to_update')
if not repo_components_to_update:
raise RuntimeError('*** Fatal error! <repo_components_to_update> not defined for: %s' % s)
repo_url_specifier = bldinstallercommon.safe_config_key_fetch(parser, s, 'repo_url_specifier')
if not repo_url_specifier:
raise RuntimeError('*** Fatal error! <repo_url_specifier> not defined for: %s' % s)
# determine full path for the conf file
full_conf_file_path = os.path.join(conf_file_base_dir, arg_configurations_file)
# create build job
job = BuildJob(is_repo_job, license_type, s, arch, version_number, version_number_tag, conf_file_base_dir, full_conf_file_path, ifw_tools_url, arg_substitution_list, repo_content_type, repo_components_to_update, repo_url_specifier, installer_name, rta_key_list)
if (job.validate()):
job_list.append(job)
return job_list
def __init__(self, section_name, target_config, packages_full_path_list, archive_location_resolver, key_value_substitution_list, is_offline_build):
self.static_component = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'static_component')
self.root_component = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'root_component')
self.package_name = section_name
self.package_subst_name = section_name
self.packages_full_path_list = packages_full_path_list
self.archives = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'archives')
self.archives = self.archives.replace(' ', '').replace('\n', '')
self.archive_server_name = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'archive_server_name')
self.downloadable_archive_list = []
self.target_install_base = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'target_install_base')
self.version = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'version')
self.version_tag = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'version_tag')
self.package_default = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'package_default')
self.install_priority = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'install_priority')
self.sorting_priority = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'sorting_priority')
self.component_sha1 = ""
self.component_sha1_uri = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'component_sha1_uri')
if (self.component_sha1_uri):
self.component_sha1_uri = archive_location_resolver.resolve_full_uri(self.package_name, self.archive_server_name, self.component_sha1_uri)
self.optional_for_offline = False
self.key_value_substitution_list = key_value_substitution_list
self.archive_skip = False
self.include_filter = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'include_filter')
if is_offline_build:
tmp = bldinstallercommon.safe_config_key_fetch(target_config, section_name, 'optional_for_offline')
for item in self.key_value_substitution_list:
tmp = tmp.replace(item[0], item[1])
if tmp.lower() in ['yes', 'true', '1']:
self.optional_for_offline = True
self.downloadable_arch_list_qs = []
self.pkg_template_dir = ''
self.sanity_check_error_msg = ''
self.target_config = target_config
self.archive_location_resolver = archive_location_resolver
# substitute key-value pairs if any
for item in self.key_value_substitution_list:
self.target_install_base = self.target_install_base.replace(item[0], item[1])
self.version = self.version.replace(item[0], item[1])
def get_job_list(optionDict, job_type_specifier, branch, arch, global_version,
global_version_tag):
conf_file = optionDict['RELEASE_DESCRIPTION_FILE']
license_type = optionDict['LICENSE']
platform = optionDict['HOST_PLATFORM']
conf_file_base_dir = optionDict['CONFIGURATIONS_FILE_BASE_DIR']
ifw_base_url = optionDict.get('IFW_TOOLS_BASE_URL', None)
print('Get [{0}] build job list for: {1}'.format(job_type_specifier,
platform + '-' + arch))
if not os.path.isfile(conf_file):
raise IOError('*** Fatal error! Given file does not exist: %s' %
conf_file)
# ensure the string ends with '/'
if ifw_base_url and not ifw_base_url.endswith('/'):
ifw_base_url += '/'
parser = ConfigParser.ConfigParser()
parser.readfp(open(conf_file))
# validate job type
if not is_valid_job_type(job_type_specifier):
raise RuntimeError(
'*** Fatal error! Unsupported job type specifier given: %s' %
job_type_specifier)
# determine which ifw tools to use (which platform)
ifw_tools = bldinstallercommon.safe_config_key_fetch(
parser, 'ifwtools', platform + '-' + arch)
if not ifw_tools:
raise RuntimeError(
'*** Fatal error! Unable to find ifw tools for_ %s' %
(platform + '-' + arch))
ifw_tools_url = urlparse.urljoin(ifw_base_url, ifw_tools)
# check if repository build job
is_repo_job = False
if job_type_specifier == 'repository':
is_repo_job = True
# first read global arg substitution list applicable for all build jobs in this file
global_arg_substitution_list = bldinstallercommon.safe_config_key_fetch(
parser, 'release.global', 'arg_substitution_list')
# parse
job_list = []
for s in parser.sections():
section_parts = s.split('.')
if (len(section_parts) < 5):
continue
if (section_parts[0] == branch
and section_parts[2] == job_type_specifier):
# Check first if this job was assigned to dedicated machine label
machine_label = bldinstallercommon.safe_config_key_fetch(
parser, s, 'assign_to_machine_label')
if machine_label:
if not machine_label == optionDict['TARGET_ENV']:
continue
else: # If not then check against the platform and arch
if not ((section_parts[3] == platform) and
(section_parts[4] == arch)):
continue
# parse from conf file
version_number = bldinstallercommon.safe_config_key_fetch(
parser, s, 'version_number')
if not version_number:
version_number = global_version
version_number_tag = bldinstallercommon.safe_config_key_fetch(
parser, s, 'version_number_tag')
if not version_number_tag:
version_number_tag = global_version_tag
arg_configurations_file = bldinstallercommon.safe_config_key_fetch(
parser, s, 'arg_configurations_file')
if not arg_configurations_file:
raise RuntimeError(
'*** Fatal error! Configuration file not defined for: %s' %
s)
# for triggering rta later on if specified
rta_key_list = bldinstallercommon.safe_config_key_fetch(
parser, s, 'rta_key_list')
# preferred installer name
installer_name = bldinstallercommon.safe_config_key_fetch(
parser, s, 'installer_name')
arg_substitution_list = bldinstallercommon.safe_config_key_fetch(
parser, s, 'arg_substitution_list')
arg_substitution_list += ',' + global_arg_substitution_list
arg_substitution_list += ',' + "%BUILD_NUMBER%=" + optionDict[
'BUILD_NUMBER']
arg_substitution_list = arg_substitution_list.replace('\n', '')
arg_substitution_list = preformat_global_version_and_tag(
arg_substitution_list, global_version, global_version_tag)
repo_content_type = ''
repo_components_to_update = ''
repo_url_specifier = ''
# if online repo job
if job_type_specifier == 'repository':
repo_content_type = bldinstallercommon.safe_config_key_fetch(
parser, s, 'repo_content_type')
if not repo_content_type:
repo_content_type = arg_configurations_file.split(
"/"
)[-1] # if the 'repo_content_type' (for temp dir name) is not defined then parse it from the conf file
repo_components_to_update = bldinstallercommon.safe_config_key_fetch(
parser, s, 'repo_components_to_update')
if not repo_components_to_update:
raise RuntimeError(
'*** Fatal error! <repo_components_to_update> not defined for: %s'
% s)
repo_url_specifier = bldinstallercommon.safe_config_key_fetch(
parser, s, 'repo_url_specifier')
if not repo_url_specifier:
raise RuntimeError(
'*** Fatal error! <repo_url_specifier> not defined for: %s'
% s)
# determine full path for the conf file
full_conf_file_path = os.path.join(conf_file_base_dir,
arg_configurations_file)
# create build job
job = BuildJob(is_repo_job, license_type, s, arch, version_number,
version_number_tag, conf_file_base_dir,
full_conf_file_path, ifw_tools_url,
arg_substitution_list, repo_content_type,
repo_components_to_update, repo_url_specifier,
installer_name, rta_key_list)
if (job.validate()):
job_list.append(job)
return job_list
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--input', required=True, help="Path to config file")
args = parser.parse_args()
if not os.path.isfile(args.input):
raise ValueError('The given path does not point into a file: ' % filePath)
configFile = open(args.input, 'r')
configParser = ConfigParser.ConfigParser()
configParser.readfp(configFile)
# base dir for pkg templates sources
baseDir = os.path.dirname(args.input)
qtVersion = bldinstallercommon.safe_config_key_fetch(configParser, 'common', 'qt_version')
qtVersionTag = bldinstallercommon.safe_config_key_fetch(configParser, 'common', 'qt_version_tag')
qtPkgName = bldinstallercommon.safe_config_key_fetch(configParser, 'common', 'qt_pkg_name') # e.g. "59" or "qt5.5100"
qt5Branch = bldinstallercommon.safe_config_key_fetch(configParser, 'common', 'qt5_branch')
# output dir
outputBaseDirName = os.path.join(currentDir, "pkg_generator_output")
qt5CheckoutDir = os.path.join(outputBaseDirName, "qt5")
pkgOutputBaseDirName = os.path.join(outputBaseDirName, "pkg")
confFilesBaseOutputDir = os.path.join(outputBaseDirName, "configurations")
shutil.rmtree(outputBaseDirName, ignore_errors=True)
os.makedirs(pkgOutputBaseDirName)
os.makedirs(qt5CheckoutDir)
os.makedirs(confFilesBaseOutputDir)
# fetch qt5:.gitmodules
modules = parseGitModules(qt5Branch, qt5CheckoutDir)
def readValueAsList(configParser, section, key):
value = bldinstallercommon.safe_config_key_fetch(configParser, section,
key)
return trimAndSplit(value.rstrip(","))
def readValueAsList(configParser, section, key):
value = bldinstallercommon.safe_config_key_fetch(configParser, section, key)
return trimAndSplit(value.rstrip(","))
def __init__(self, section_name, target_config, packages_full_path_list,
archive_location_resolver, key_value_substitution_list,
is_offline_build):
self.static_component = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'static_component')
self.root_component = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'root_component')
self.package_name = section_name
self.package_subst_name = section_name
self.packages_full_path_list = packages_full_path_list
self.archives = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'archives')
self.archives = self.archives.replace(' ', '').replace('\n', '')
self.archive_server_name = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'archive_server_name')
self.downloadable_archive_list = []
self.target_install_base = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'target_install_base')
self.version = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'version')
self.version_tag = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'version_tag')
self.package_default = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'package_default')
self.install_priority = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'install_priority')
self.sorting_priority = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'sorting_priority')
self.component_sha1 = ""
self.component_sha1_uri = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'component_sha1_uri')
if (self.component_sha1_uri):
self.component_sha1_uri = archive_location_resolver.resolve_full_uri(
self.package_name, self.archive_server_name,
self.component_sha1_uri)
self.optional_for_offline = False
self.key_value_substitution_list = key_value_substitution_list
self.archive_skip = False
self.include_filter = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'include_filter')
if is_offline_build:
tmp = bldinstallercommon.safe_config_key_fetch(
target_config, section_name, 'optional_for_offline')
for item in self.key_value_substitution_list:
tmp = tmp.replace(item[0], item[1])
if tmp.lower() in ['yes', 'true', '1']:
self.optional_for_offline = True
self.downloadable_arch_list_qs = []
self.pkg_template_dir = ''
self.sanity_check_error_msg = ''
self.target_config = target_config
self.archive_location_resolver = archive_location_resolver
# substitute key-value pairs if any
for item in self.key_value_substitution_list:
self.target_install_base = self.target_install_base.replace(
item[0], item[1])
self.version = self.version.replace(item[0], item[1])
def get_job_list(optionDict, job_type_specifier, branch, arch, global_version, global_version_tag):
conf_file = optionDict["RELEASE_DESCRIPTION_FILE"]
license_type = optionDict["LICENSE"]
platform = optionDict["HOST_PLATFORM"]
conf_file_base_dir = optionDict["CONFIGURATIONS_FILE_BASE_DIR"]
ifw_base_url = optionDict["IFW_TOOLS_BASE_URL"]
print("Get [{0}] build job list for: {1}".format(job_type_specifier, platform + "-" + arch))
if not os.path.isfile(conf_file):
raise IOError("*** Fatal error! Given file does not exist: %s" % conf_file)
# ensure the string ends with '/'
if not ifw_base_url.endswith("/"):
ifw_base_url += "/"
parser = ConfigParser.ConfigParser()
parser.readfp(open(conf_file))
# validate job type
if not is_valid_job_type(job_type_specifier):
raise RuntimeError("*** Fatal error! Unsupported job type specifier given: %s" % job_type_specifier)
# determine which ifw tools to use (which platform)
ifw_tools = bldinstallercommon.safe_config_key_fetch(parser, "ifwtools", platform + "-" + arch)
if not ifw_tools:
raise RuntimeError("*** Fatal error! Unable to find ifw tools for_ %s" % (platform + "-" + arch))
ifw_tools_url = urlparse.urljoin(ifw_base_url, ifw_tools)
# check if repository build job
is_repo_job = False
if job_type_specifier == "repository":
is_repo_job = True
# first read global arg substitution list applicable for all build jobs in this file
global_arg_substitution_list = bldinstallercommon.safe_config_key_fetch(
parser, "release.global", "arg_substitution_list"
)
# parse
job_list = []
for s in parser.sections():
section_parts = s.split(".")
if len(section_parts) < 5:
continue
if section_parts[0] == branch and section_parts[2] == job_type_specifier:
# Check first if this job was assigned to dedicated machine label
machine_label = bldinstallercommon.safe_config_key_fetch(parser, s, "assign_to_machine_label")
if machine_label:
if not machine_label == optionDict["TARGET_ENV"]:
continue
else: # If not then check against the platform and arch
if not ((section_parts[3] == platform) and (section_parts[4] == arch)):
continue
# parse from conf file
version_number = bldinstallercommon.safe_config_key_fetch(parser, s, "version_number")
if not version_number:
version_number = global_version
version_number_tag = bldinstallercommon.safe_config_key_fetch(parser, s, "version_number_tag")
if not version_number_tag:
version_number_tag = global_version_tag
arg_configurations_file = bldinstallercommon.safe_config_key_fetch(parser, s, "arg_configurations_file")
if not arg_configurations_file:
raise RuntimeError("*** Fatal error! Configuration file not defined for: %s" % s)
# for triggering rta later on if specified
rta_key_list = bldinstallercommon.safe_config_key_fetch(parser, s, "rta_key_list")
# preferred installer name
installer_name = bldinstallercommon.safe_config_key_fetch(parser, s, "installer_name")
arg_substitution_list = bldinstallercommon.safe_config_key_fetch(parser, s, "arg_substitution_list")
arg_substitution_list += "," + global_arg_substitution_list
arg_substitution_list = arg_substitution_list.replace("\n", "")
arg_substitution_list = preformat_global_version_and_tag(
arg_substitution_list, global_version, global_version_tag
)
repo_content_type = ""
repo_components_to_update = ""
repo_url_specifier = ""
# if online repo job
if job_type_specifier == "repository":
repo_content_type = bldinstallercommon.safe_config_key_fetch(parser, s, "repo_content_type")
if not repo_content_type:
raise RuntimeError("*** Fatal error! <repo_content_type> not defined for: %s" % s)
repo_components_to_update = bldinstallercommon.safe_config_key_fetch(
parser, s, "repo_components_to_update"
)
if not repo_components_to_update:
raise RuntimeError("*** Fatal error! <repo_components_to_update> not defined for: %s" % s)
repo_url_specifier = bldinstallercommon.safe_config_key_fetch(parser, s, "repo_url_specifier")
if not repo_url_specifier:
raise RuntimeError("*** Fatal error! <repo_url_specifier> not defined for: %s" % s)
# determine full path for the conf file
full_conf_file_path = os.path.join(conf_file_base_dir, arg_configurations_file)
# create build job
job = BuildJob(
is_repo_job,
license_type,
s,
arch,
version_number,
version_number_tag,
conf_file_base_dir,
full_conf_file_path,
ifw_tools_url,
arg_substitution_list,
repo_content_type,
repo_components_to_update,
repo_url_specifier,
installer_name,
rta_key_list,
)
if job.validate():
job_list.append(job)
return job_list
def handle_repo_build(optionDict, branch, arch, update_staging_repo, update_production_repo):
conf_file = optionDict['RELEASE_DESCRIPTION_FILE']
if not os.path.isfile(conf_file):
raise IOError('*** Fatal error! Given file does not exist: %s' % conf_file)
init_env(optionDict)
packages_base_url = optionDict['PACKAGE_STORAGE_SERVER_PATH_HTTP']
ifw_base_url = optionDict['IFW_TOOLS_BASE_URL']
release_tools_dir = SCRIPT_ROOT_DIR
# parse conf file
parser = ConfigParser.ConfigParser()
parser.readfp(open(conf_file))
section_name = branch + '.' + 'global'
global_version = bldinstallercommon.safe_config_key_fetch(parser, section_name, 'version')
global_version_tag = bldinstallercommon.safe_config_key_fetch(parser, section_name, 'version_tag')
# parse build jobs
repo_job_list = get_repo_job_list(optionDict, branch, arch, global_version, global_version_tag)
if (len(repo_job_list) == 0):
raise RuntimeError('*** Fatal error! No repository build jobs found. Probably an error? %s' % conf_file)
# init repo dirs
init_repositories(optionDict, repo_job_list)
# is this snapshot build? Then enable component version number forced update
forced_version_number_bump = False
if update_staging_repo and not update_production_repo:
forced_version_number_bump = True
if optionDict.get('FORCE_VERSION_NUMBER_INCREASE') in ['yes', 'true', '1']:
forced_version_number_bump = True
# create rta description file
rta_descr_output_dir = os.path.join(SCRIPT_ROOT_DIR, pkg_constants.RTA_DESCRIPTION_FILE_DIR_NAME)
bldinstallercommon.create_dirs(rta_descr_output_dir)
architecture = bldinstallercommon.get_architecture()
plat_suffix = bldinstallercommon.get_platform_suffix()
rta_description_file_name = os.path.join( rta_descr_output_dir, pkg_constants.RTA_DESCRIPTION_FILE_NAME_BASE + '-' + plat_suffix + '-' + architecture + '-repo.txt')
# handle repo build jobs
for job in repo_job_list:
create_online_repository(job, packages_base_url, forced_version_number_bump)
# determine testination path on test server
dest_path_repository, dest_path_pkg = generate_repo_dest_path_pending(optionDict, job)
# copy repo content to test server
source_path_repository = os.path.join(release_tools_dir, 'online_repository')
source_path_pkg = os.path.join(release_tools_dir, 'pkg')
push_online_repository(optionDict, optionDict['PKG_STAGING_SERVER'], optionDict['PKG_STAGING_SERVER_UNAME'], source_path_repository, dest_path_repository)
push_online_repository(optionDict, optionDict['PKG_STAGING_SERVER'], optionDict['PKG_STAGING_SERVER_UNAME'], source_path_pkg, dest_path_pkg)
# remove local repository and pkg directories
bldinstallercommon.remove_tree(source_path_repository)
bldinstallercommon.remove_tree(source_path_pkg)
# update repo in testing area
staging_repo_updated, production_repo_updated = update_online_repo(optionDict, job, update_staging_repo, update_production_repo)
# write the rta description file only if staging repository creation was ok
# remove also temp staging repositories from 'staging_pending' and 'production_dist_update_work' directories
server_addr = optionDict['PKG_STAGING_SERVER_UNAME'] + '@' + optionDict['PKG_STAGING_SERVER']
if (staging_repo_updated):
rta_description_file = open(rta_description_file_name, 'a')
rta_description_file.write(job.repo_url_specifier + ' ' + job.rta_key_list + '\n')
rta_description_file.close()
staging_temp_content_to_be_deleted = generate_repo_path_for_pending_area(optionDict, job)
delete_online_repo_paths(optionDict, server_addr, staging_temp_content_to_be_deleted)
if (production_repo_updated):
production_temp_content_to_be_deleted = optionDict['REPO_STAGING_SERVER_TEST_REPO_DIST_WORK'] + '/' + optionDict['ONLINE_REPOSITORY_BASE_NAME'] + '/' + job.repo_url_specifier
delete_online_repo_paths(optionDict, server_addr, production_temp_content_to_be_deleted)
def handle_repo_build(optionDict, branch, arch, update_staging_repo, update_production_repo):
conf_file = optionDict["RELEASE_DESCRIPTION_FILE"]
if not os.path.isfile(conf_file):
raise IOError("*** Fatal error! Given file does not exist: %s" % conf_file)
init_env(optionDict)
packages_base_url = optionDict["PACKAGE_STORAGE_SERVER_PATH_HTTP"]
ifw_base_url = optionDict["IFW_TOOLS_BASE_URL"]
release_tools_dir = SCRIPT_ROOT_DIR
# parse conf file
parser = ConfigParser.ConfigParser()
parser.readfp(open(conf_file))
section_name = branch + "." + "global"
global_version = bldinstallercommon.safe_config_key_fetch(parser, section_name, "version")
global_version_tag = bldinstallercommon.safe_config_key_fetch(parser, section_name, "version_tag")
# parse build jobs
repo_job_list = get_repo_job_list(optionDict, branch, arch, global_version, global_version_tag)
if len(repo_job_list) == 0:
raise RuntimeError("*** Fatal error! No repository build jobs found. Probably an error? %s" % conf_file)
# init repo dirs
init_repositories(optionDict, repo_job_list)
# is this snapshot build? Then enable component version number forced update
forced_version_number_bump = False
if update_staging_repo and not update_production_repo:
forced_version_number_bump = True
# create rta description file
rta_descr_output_dir = os.path.join(SCRIPT_ROOT_DIR, pkg_constants.RTA_DESCRIPTION_FILE_DIR_NAME)
bldinstallercommon.create_dirs(rta_descr_output_dir)
architecture = bldinstallercommon.get_architecture()
plat_suffix = bldinstallercommon.get_platform_suffix()
rta_description_file_name = os.path.join(
rta_descr_output_dir,
pkg_constants.RTA_DESCRIPTION_FILE_NAME_BASE + "-" + plat_suffix + "-" + architecture + "-repo.txt",
)
# handle repo build jobs
for job in repo_job_list:
create_online_repository(job, packages_base_url, forced_version_number_bump)
# determine testination path on test server
dest_path_repository, dest_path_pkg = generate_repo_dest_path_pending(optionDict, job)
# copy repo content to test server
source_path_repository = os.path.join(release_tools_dir, "repository")
source_path_pkg = os.path.join(release_tools_dir, "pkg")
push_online_repository(
optionDict,
optionDict["PKG_STAGING_SERVER"],
optionDict["PKG_STAGING_SERVER_UNAME"],
source_path_repository,
dest_path_repository,
)
push_online_repository(
optionDict,
optionDict["PKG_STAGING_SERVER"],
optionDict["PKG_STAGING_SERVER_UNAME"],
source_path_pkg,
dest_path_pkg,
)
# remove local repository and pkg directories
bldinstallercommon.remove_tree(source_path_repository)
bldinstallercommon.remove_tree(source_path_pkg)
# update repo in testing area
staging_repo_updated, dummy = update_online_repo(optionDict, job, update_staging_repo, update_production_repo)
# write the rta description file only if staging repository creation was ok
if staging_repo_updated:
rta_description_file = open(rta_description_file_name, "a")
rta_description_file.write(job.repo_url_specifier + " " + job.rta_key_list + "\n")
rta_description_file.close()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--input', required=True, help="Path to config file")
args = parser.parse_args()
if not os.path.isfile(args.input):
raise ValueError('The given path does not point into a file: ' %
filePath)
configFile = open(args.input, 'r')
configParser = ConfigParser.ConfigParser()
configParser.readfp(configFile)
# base dir for pkg templates sources
baseDir = os.path.dirname(args.input)
qtVersion = bldinstallercommon.safe_config_key_fetch(
configParser, 'common', 'qt_version')
qtVersionTag = bldinstallercommon.safe_config_key_fetch(
configParser, 'common', 'qt_version_tag')
qtPkgName = bldinstallercommon.safe_config_key_fetch(
configParser, 'common', 'qt_pkg_name') # e.g. "59" or "qt5.5100"
qt5Branch = bldinstallercommon.safe_config_key_fetch(
configParser, 'common', 'qt5_branch')
# output dir
outputBaseDirName = os.path.join(currentDir, "pkg_generator_output")
qt5CheckoutDir = os.path.join(outputBaseDirName, "qt5")
pkgOutputBaseDirName = os.path.join(outputBaseDirName, "pkg")
confFilesBaseOutputDir = os.path.join(outputBaseDirName, "configurations")
shutil.rmtree(outputBaseDirName, ignore_errors=True)
os.makedirs(pkgOutputBaseDirName)
os.makedirs(qt5CheckoutDir)
os.makedirs(confFilesBaseOutputDir)
