def build_via_custom_script(self, module):
"""Function that builds a module using its custom build script
Parameters
----------
module : InstallModule
module to build with custom build script
Returns
-------
int
exit code of custom build script
"""
current = os.getcwd()
os.chdir(module.abs_path)
if platform == 'win32':
exec = module.custom_build_script_path
else:
exec = 'bash {}'.format(module.custom_build_script_path)
LOG.print_command(exec)
proc = Popen(exec.split(' '))
proc.wait()
ret = proc.returncode
os.chdir(current)
return ret
def acquire_dependecies(self, dependency_script_path):
"""Method that runs dependency install shell/batch script
Parameters
----------
dependency_script_path : str
path to dependency shell/batch script
"""
LOG.debug('Grabbing dependencies via script {}'.format(
dependency_script_path))
if os.path.exists(dependency_script_path) and os.path.isfile(
dependency_script_path):
if dependency_script_path.endswith('.bat'):
exec = dependency_script_path
else:
exec = 'bash {}'.format(dependency_script_path)
LOG.print_command(exec)
proc = Popen(exec.split(' '))
proc.wait()
ret = proc.returncode
if ret != 0:
LOG.write(
'Dependency script exited with non-zero exit code: {}'.
format(ret))
def build_module(self, module_name):
"""Function that executes build of single module
First, checks if all dependencies built, if not, does that first.
Then checks for custom build script. If one is found, runs that from
module root directory.
Otherwise, runs make followed by specified make flag in module root
directory.
Parameters
----------
module_name : str
The name of the module being built
Returns
-------
int
The return code of the build process, 0 if module is not buildable (ex. UTILS)
"""
if module_name in self.non_build_packages:
return 0
LOG.write('Building module {}'.format(module_name))
module = self.install_config.get_module_by_name(module_name)
if len(module.dependencies) > 0:
for dep in module.dependencies:
if dep not in self.built:
self.build_module(dep)
if module.custom_build_script_path is not None:
LOG.write('Detected custom build script located at {}'.format(
module.custom_build_script_path))
ret = self.build_via_custom_script(module)
if ret == 0:
self.built.append(module_name)
LOG.write(
'Built module {} via custom script'.format(module_name))
else:
LOG.write(
'Custom script for module {} exited with error code {}.'.
format(module_name, ret))
else:
command = "make -C {} {}".format(module.abs_path, self.make_flag)
LOG.print_command(command)
proc = Popen(command.split(' '))
proc.wait()
ret = proc.returncode
if ret == 0:
self.built.append(module_name)
LOG.write('Built module {}'.format(module_name))
else:
LOG.write('Failed to build module {}'.format(module_name))
return ret
def make_support_releases_consistent(self):
"""Function that makes support module release files consistent
Returns
-------
int
return code of make release command call
"""
LOG.write('Running make release to keep releases consistent.')
command = 'make -C {} release'.format(self.install_config.support_path)
LOG.print_command(command)
proc = Popen(command.split(' '))
proc.wait()
ret = proc.returncode
if ret != 0:
LOG.write(
'make release exited with non-zero exit code: {}'.format(ret))
return ret
def update_submodule(self, module, submodule_name):
"""Function that updates submodules given that the input module is in the self.submodule_list array
Parameters
----------
module : InstallModule
module for which we must update submodules
submodule_name : str
name of submodule to update
"""
LOG.debug('Updating git submodules for {}'.format(module.name))
if isinstance(module, IM.InstallModule):
if module.abs_path != None:
submodule_path = module.abs_path + "/" + submodule_name
if os.path.exists(submodule_path):
LOG.print_command(
'git -C {} submodule init'.format(submodule_path))
p1 = Popen(
["git", "-C", submodule_path, "submodule", "init"])
p1.wait()
ret1 = p1.returncode
if ret1 == 0:
LOG.debug(
'Submodules initialized for module {}.'.format(
module.name))
else:
LOG.debug(
'Failed to initialize submodules for module {}.'.
format(module.name))
LOG.print_command(
'git -C {} submodule update'.format(submodule_path))
p2 = Popen(
["git", "-C", submodule_path, "submodule", "update"])
p2.wait()
ret2 = p2.returncode
if ret2 == 0:
LOG.debug('Submodules updated for module {}.'.format(
module.name))
else:
LOG.debug('Failed to update submodules for module {}.'.
format(module.name))
def checkout_module(self, module):
"""Function responsible for checking out selected tagged versions of modules.
Parameters
----------
module : InstallModule
Module that is being checked out
Returns
-------
int
-3 if input was not an InstallModule, -2 if the absolute path is not known, -1 if checkout fails, 0 if success
"""
ret = -1
LOG.debug('Checking out version for module {}'.format(module.name))
if isinstance(module, IM.InstallModule):
if module.abs_path != None:
ret = 0
if module.version != "master" and module.url_type == "GIT_URL":
current_loc = os.getcwd()
os.chdir(module.abs_path)
command = "git checkout -q {}".format(module.version)
LOG.print_command(command)
proc = Popen(command.split(' '))
proc.wait()
ret = proc.returncode
os.chdir(current_loc)
if ret == 0:
LOG.write('Checked out version {}'.format(
module.version))
else:
LOG.write(
'Checkout of version {} failed for module {}.'.
format(module.version, module.name))
return ret
def sync_module_tag(module_name, install_config, save_path=None):
"""Function that syncs module version tags with those hosted with git.
This function is still buggy, and certain modules do not update correctly
Parameters
----------
module_name : str
The name of the module to sync
install_config : InstallConfiguration
instance of install configuration for which to update tags
save_path : str
None by default. If set, will save the install configuration to the given location after updating.
"""
module = install_config.get_module_by_name(module_name)
if module.url_type == 'GIT_URL' and module.version != 'master' and module.name not in update_tags_blacklist:
account_repo = '{}{}'.format(module.url, module.repository)
LOG.print_command("git ls-remote --tags {}".format(account_repo))
sync_tags_proc = Popen(['git', 'ls-remote', '--tags', account_repo],
stdout=PIPE,
stderr=PIPE)
out, err = sync_tags_proc.communicate()
ret = out.decode('utf-8')
tags_temp = ret.splitlines()
tags = []
for tag in tags_temp:
tags.append(tag.rsplit('/')[-1])
if len(tags) > 0:
best_tag = tags[0]
best_tag_ver_str_list = re.split(r'\D+', tags[0])
best_tag_ver_str_list = [
num for num in best_tag_ver_str_list if num.isnumeric()
]
best_tag_version_numbers = list(map(int, best_tag_ver_str_list))
for tag in tags:
tag_ver_str_list = re.split(r'\D+', tag)
tag_ver_str_list = [
num for num in tag_ver_str_list if num.isnumeric()
]
tag_version_numbers = list(map(int, tag_ver_str_list))
for i in range(len(tag_version_numbers)):
if best_tag.startswith('R') and not tag.startswith('R'):
break
elif not best_tag.startswith('R') and tag.startswith('R'):
best_tag = tag
best_tag_version_numbers = tag_version_numbers
break
elif i == len(
best_tag_version_numbers
) or tag_version_numbers[i] > best_tag_version_numbers[i]:
best_tag = tag
best_tag_version_numbers = tag_version_numbers
break
elif tag_version_numbers[i] < best_tag_version_numbers[i]:
break
tag_updated = False
module_ver_str_list = re.split(r'\D+', module.version)
module_ver_str_list = [
num for num in module_ver_str_list if num.isnumeric()
]
module_version_numbers = list(map(int, module_ver_str_list))
for i in range(len(best_tag_version_numbers)):
if i == len(
module_version_numbers
) or best_tag_version_numbers[i] > module_version_numbers[i]:
tag_updated = True
LOG.write(
'Updating {} from version {} to version {}'.format(
module.name, module.version, best_tag))
module.version = best_tag
break
elif best_tag_version_numbers[i] < module_version_numbers[i]:
break
if not tag_updated:
LOG.debug('Module {} already at latest version: {}'.format(
module.name, module.version))
if save_path is not None:
writer = IO.config_writer.ConfigWriter(install_config)
ret, message = writer.write_install_config(save_path,
overwrite_existing=True)
LOG.write('Updated install config saved to {}'.format(save_path))
return ret
else:
return True
def clone_module(self, module, recursive=False):
"""Function responsible for cloning each module into the appropriate location
First checks if the module uses git or a download, and whether it needs to be recursive
then, uses the information in the module object along with subprocess commands to clone the module.
Parameters
----------
module : InstallModule
InstallModule currently being cloned
recursive=False
Flag that decides if git clone should be done recursively
"""
LOG.debug('Cloning module {}'.format(module.name))
if isinstance(module, IM.InstallModule):
if module.abs_path != None:
ret = -1
if os.path.exists(module.abs_path):
shutil.rmtree(module.abs_path)
if not recursive and module.url_type == "GIT_URL":
command = "git clone {} {}".format(
module.url + module.repository, module.abs_path)
elif recursive and module.url_type == "GIT_URL":
command = "git clone --recursive {} {}".format(
module.url + module.repository, module.abs_path)
elif module.url_type == "WGET_URL":
if platform == "win32":
command = "wget --no-check-certificate -P {} {}".format(
module.abs_path, module.url + module.repository)
else:
command = 'wget -P {} {}'.format(
module.abs_path, module.url + module.repository)
LOG.print_command(command)
proc = Popen(command.split(' '))
proc.wait()
ret = proc.returncode
if ret == 0:
LOG.write('Cloned module {} successfully.'.format(
module.name))
else:
LOG.write('Failed to clone module {}.'.format(module.name))
return -1
if module.url_type == "WGET_URL":
if (module.repository.endswith(".tar.gz") or
module.repository.endswith(".tgz")) and ret == 0:
command = "tar -xzf {} -C {} --strip-components=1".format(
os.path.join(module.abs_path, module.repository),
module.abs_path)
elif module.repository.endswith(".zip") and ret == 0:
command = "unzip {} -C {}".format(
os.path.join(module.abs_path, module.repository),
module.abs_path)
LOG.print_command(command)
proc = Popen(command.split(' '))
proc.wait()
ret = proc.returncode
if ret == 0:
LOG.write('Unpacked module {} successfully.'.format(
module.name))
else:
LOG.write('Failed to unpack module {}.'.format(
module.name))
if ret == 0:
return ret
return -1
return -2
return -3