def download(self, name, root): # pylint: disable=R0201,R0915 assert name is not None assert root is not None logging.debug('Trying download %s in folder %s', name, root) if name not in self.packages.keys(): debug_red('The package %s is not in the package file.', name) return None package = self.packages[name] fetcher = package['fetch'] self.extract_queue = [] if isinstance(fetcher, list): for fetch in fetcher: if self.download_methods[fetch](name, package, root) is None: return None elif isinstance(fetcher, str): if self.download_methods[fetcher](name, package, root) is None: return None else: debug_red( 'The fetcher field of the package \'%s\' is invalid: %s', name, fetcher, ) return None self._try_extract(package) return 0
def execute(self, name): assert name is not None build_dir = self.root extra_targets = [] self.get_optional('gradle_targets', extra_targets.extend) for target in extra_targets: gradle_command = ['gradle'] self.append_optional('gradle_args', gradle_command) gradle_command.append(target) logging.debug('Running gradle with: %s', ' '.join(gradle_command)) with output_header('Gradle'): child = subprocess.Popen( gradle_command, cwd=build_dir, ) _ = child.communicate()[0] ret_code = child.returncode if ret_code != 0: debug_red('Running gradle target %s failed', target) return None return 0
def execute(self, name): assert name is not None cmake_command = ['cmake'] cmake_command.append('-S') cmake_command.append(self.root) self.append_optional('cmake_args', cmake_command) if 'DCMAKE_INSTALL_PREFIX' not in cmake_command: cmake_command.append(f'-DCMAKE_INSTALL_PREFIX={self.usr_dir}') build_dir = os.path.join(self.root, 'build') if not os.path.isdir(build_dir): os.makedirs(build_dir) logging.debug('Running cmake with: %s', ' '.join(cmake_command)) logging.debug('Build directory: %s', build_dir) with output_header('CMake'): child = subprocess.Popen( cmake_command, cwd=build_dir, ) _ = child.communicate()[0] ret_code = child.returncode if ret_code != 0: debug_red('Running cmake failed') return None return 0
def execute(self, name): assert name is not None build_dir = self.root extra_scripts = [] self.get_optional('nodejs_scripts', extra_scripts.extend) for target in extra_scripts: nodejs_command = ['nodejs'] self.append_optional('nodejs_args', nodejs_command) nodejs_command.append(target) logging.debug('Running make with: %s', ' '.join(nodejs_command)) with output_header('NPM'): child = subprocess.Popen( nodejs_command, cwd=build_dir, ) _ = child.communicate()[0] ret_code = child.returncode if ret_code != 0: debug_red('Running npm script %s failed', target) return None return 0
def _add_installer(self, installer, file): assert installer is not None assert file is not None if not hasattr(installer, 'ExportedClass'): debug_red('No exported class found in file %s', file) return if issubclass(installer.ExportedClass, BasicInstaller) is None: debug_red( 'The exported class is not a subclass of BasicInstaller.') return if installer.ExportedClass.name is None: debug_red('The exported class does not have proper name.') return InstallerClass = installer.ExportedClass # pylint: disable=C0103 if InstallerClass.name in self.installers.keys(): debug_red( 'Installer with the name \'%s\' already exists', InstallerClass.name, ) debug_cyan('Loading installer: \'%s\'', InstallerClass.name) self.installers[InstallerClass.name] = InstallerClass
def _download_curl(self, name, package, root): # pylint: disable=R0201 logging.info('Trying to fetch with curl.') if 'curl' not in package.keys() or not isinstance( package['curl'], dict): debug_red('Invalid curl node for packag %s.', name) return None if 'url' not in package['curl'].keys(): debug_red('The git node of %s does not have urlf field.', name) return None curl_node = package['curl'] url = curl_node['url'] path = root file_name = file_name = url.split('/')[-1] with suppress(OSError): os.makedirs(path) cmd = [] cmd.append(self.CURL_COMMAND) if 'args' in curl_node.keys() and isinstance(curl_node['args'], str): cmd.append(curl_node['args']) cmd.append(url) if 'output' in curl_node.keys() and isinstance(curl_node['output'], str): cmd.append('-o') cmd.append(curl_node['output']) file_name = curl_node['output'] else: cmd.append('-O') logging.debug('Fetching with curl and command: %s', cmd) if execute_sanitized('curl', cmd, path) is None: return None file_path = os.path.join(path, file_name) for ext in self.archive_extensions: if file_path.endswith(ext): self.extract_queue.append((file_path, name)) break return 0
def checkout(self, git_node, path): if 'checkout' in git_node.keys() and isinstance( git_node['checkout'], str): cmd = [] cmd.append(self.GIT_COMMAND) cmd.append('checkout') cmd.append(git_node['checkout']) logging.debug( 'Checking out a git repository with "%s" ', ' '.join(cmd), ) res = subprocess.run(cmd, cwd=path) if res.returncode != 0: debug_red('The checking out failed!') return None return 0
def execute(self, name): assert name is not None bash_lines = self.node['bash_lines'] in_bashrc = self.node.get('in_bashrc', False) if isinstance(bash_lines, str): bash_lines = [sanitize_input_variable(bash_lines)] else: bash_lines = map(sanitize_input_variable, bash_lines) target_file = '~/.bashrc' if in_bashrc else os.path.join( self.code_dir, 'setenv.sh', ) target_file = sanitize_input_variable(target_file) with open(target_file) as target: target_file_lines = target.readlines() with open(target_file, 'a') as target: for line in bash_lines: if not BashrcInstaller._check_line(line): debug_red( 'Skipping bash line.\ The line is invalid: %s', line, ) continue if line in target_file_lines: debug_red( 'Skipping bash line.\ The line is already in the file: %s', line, ) continue logging.debug('Writing bashrc line in %s', target_file) target.write(line) return 0
def execute(self, name): assert name is not None setup_file = os.path.join(self.root, 'setup.py') if not os.path.isfile(setup_file): debug_red( 'There isn\'t a setup.py file at the root of the package %s.', name, ) return None setup_command = ['python'] setup_command.append('setup.py') self.append_optional('setup_args', setup_command) setup_command.append('install') if self.node.get('prefix', False): setup_command.append('--prefix') setup_command.append(self.usr_dir) self.append_optional('setup_install_args', setup_command) logging.debug('Running setup.py with: %s', ' '.join(setup_command)) with output_header('Setup.py'): child = subprocess.Popen( setup_command, cwd=self.root, ) _ = child.communicate()[0] ret_code = child.returncode if ret_code != 0: debug_red('Running setup.py failed') return None return 0
def execute(self, name): assert name is not None paths = self.node['path'] in_bashrc = self.node.get('in_bashrc', False) if isinstance(paths, str): paths = [paths] paths = map( lambda p: 'export LD_PATH=${{LD_PATH}}:{}'.format( sanitize_input_variable(p), ), paths, ) target_file = '~/.bashrc' if in_bashrc else os.path.join( self.code_dir, 'setenv.sh', ) target_file = sanitize_input_variable(target_file) with open(target_file) as target: target_file_lines = target.read().splitlines() with open(target_file, 'a') as target: for line in list(paths): if line in target_file_lines: debug_red( 'Skipping library path export line.\ The line is already in the file: %s', line, ) continue logging.debug('Adding library export in %s', target_file) target.write(line) target.write('\n') return 0
def execute_sanitized(name, command, cwd, supress_output=False): assert name is not None assert command is not None assert cwd is not None command = list(map(sanitize_input_variable, command)) stdout = subprocess.PIPE if supress_output else None with output_header(name): # TODO: This is very dangerous wiht shell=True # Think of something better at some point command = ' '.join(command) if isinstance(command, list) else command child = subprocess.Popen( command, cwd=cwd, stdout=stdout, shell=True, ) _ = child.communicate()[0] ret_code = child.returncode if ret_code != 0: debug_red('Running %s failed', name) return None return 0
def get_output(name, command, cwd, supress_output=False): assert name is not None assert command is not None assert cwd is not None command = list(map(sanitize_input_variable, command)) with output_header(name): child = subprocess.Popen( command, cwd=cwd, stdout=subprocess.PIPE, ) for line in child.stdout: if not supress_output: print(line) yield line _ = child.communicate()[0] ret_code = child.returncode if ret_code != 0: debug_red('Running %s failed', name) return None return 0
def execute(self, name): assert name is not None make_command = ['make'] self.append_optional('make_args', make_command) build_dir = os.path.join(self.root, 'build/') if not os.path.isdir(build_dir): build_dir = self.root if not os.path.isdir(build_dir): debug_red('There is no build directory. Connot run make') return None logging.debug('Running make with: %s', ' '.join(make_command)) logging.debug('Build directory: %s', build_dir) with output_header('Make'): child = subprocess.Popen( make_command, cwd=build_dir, ) _ = child.communicate()[0] ret_code = child.returncode if ret_code != 0: debug_red('Running make failed') return None extra_targets = [] self.get_optional('make_extra_targets', extra_targets.extend) for target in extra_targets: make_command = ['make'] self.append_optional('make_args', make_command) make_command.append(target) logging.debug('Running make with: %s', ' '.join(make_command)) with output_header('Make'): child = subprocess.Popen( make_command, cwd=build_dir, ) _ = child.communicate()[0] ret_code = child.returncode if ret_code != 0: debug_red('Running make target %s failed', target) return None return 0
def _add_command(self, command, file): assert command is not None assert file is not None if not hasattr(command, 'ExportedClass'): debug_red('No exported command class found in file %s', file) return if command.ExportedClass.name is None: debug_red('The exported class does not have proper name.') return CommandClass = command.ExportedClass # pylint: disable=C0103 if CommandClass.name in self.commands.keys(): debug_red( 'Command with the name \'%s\' already exists', CommandClass.name, ) debug_cyan('Loading command: \'%s\'', CommandClass.name) self.commands[CommandClass.name] = CommandClass
def _download_git(self, name, package, root): # pylint: disable=R0201,R0911 assert name is not None assert root is not None assert package is not None logging.info('Trying to fetch with git.') if 'git' not in package.keys() or not isinstance(package['git'], dict): debug_red('Invalid git node for packag %s.', name) return None if 'url' not in package['git'].keys(): debug_red('The git node of %s does not have urlf field.', name) return None git_node = package['git'] url = git_node['url'] # TODO: Do not join the root here but pass it from above path = root cmd = [] cmd.extend([self.GIT_COMMAND, 'clone']) if 'args' in git_node.keys() and isinstance(git_node['args'], str): cmd.append(git_node['args']) if self.git_ssh: logging.debug('Trying to use ssh with git') match = self.GIT_HTTPS_RE.match(url) if match: url = 'git@' + match.group(1) + ':' + match.group(2) cmd.append(url) elif self.GIT_SSH_RE.match(url) is not None: cmd.append(url) else: debug_red('Bad git url for %s: %s', name, url) return None else: logging.debug('Trying to use https with git') match = self.GIT_SSH_RE.match(url) if match: url = 'https://' + match.group(1) + '/' + match.group(2) cmd.append(url) elif self.GIT_HTTPS_RE.match(url) is not None: cmd.append(url) else: debug_red('Bad git url for %s: %s', name, url) return None if os.path.exists(path): debug_red('The given path already exists: %s', path) return None cmd.append(path) logging.debug('Fetching with git and command: %s', cmd) if subprocess.call(cmd) != 0: debug_red('The fetching failed!') return None # TODO: Mark somehow that the contents have been cloned but the step is not fully complete if self.checkout(git_node, path) is None: return None return 0