def download(self, name, root): # pylint: disable=R0201,R0915
assert name is not None
assert root is not None
logging.debug('Trying download %s in folder %s', name, root)
if name not in self.packages.keys():
debug_red('The package %s is not in the package file.', name)
return None
package = self.packages[name]
fetcher = package['fetch']
self.extract_queue = []
if isinstance(fetcher, list):
for fetch in fetcher:
if self.download_methods[fetch](name, package, root) is None:
return None
elif isinstance(fetcher, str):
if self.download_methods[fetcher](name, package, root) is None:
return None
else:
debug_red(
'The fetcher field of the package \'%s\' is invalid: %s',
name,
fetcher,
)
return None
self._try_extract(package)
return 0
def execute(self, name):
assert name is not None
build_dir = self.root
extra_targets = []
self.get_optional('gradle_targets', extra_targets.extend)
for target in extra_targets:
gradle_command = ['gradle']
self.append_optional('gradle_args', gradle_command)
gradle_command.append(target)
logging.debug('Running gradle with: %s', ' '.join(gradle_command))
with output_header('Gradle'):
child = subprocess.Popen(
gradle_command,
cwd=build_dir,
)
_ = child.communicate()[0]
ret_code = child.returncode
if ret_code != 0:
debug_red('Running gradle target %s failed', target)
return None
return 0
def execute(self, name):
assert name is not None
cmake_command = ['cmake']
cmake_command.append('-S')
cmake_command.append(self.root)
self.append_optional('cmake_args', cmake_command)
if 'DCMAKE_INSTALL_PREFIX' not in cmake_command:
cmake_command.append(f'-DCMAKE_INSTALL_PREFIX={self.usr_dir}')
build_dir = os.path.join(self.root, 'build')
if not os.path.isdir(build_dir):
os.makedirs(build_dir)
logging.debug('Running cmake with: %s', ' '.join(cmake_command))
logging.debug('Build directory: %s', build_dir)
with output_header('CMake'):
child = subprocess.Popen(
cmake_command,
cwd=build_dir,
)
_ = child.communicate()[0]
ret_code = child.returncode
if ret_code != 0:
debug_red('Running cmake failed')
return None
return 0
def execute(self, name):
assert name is not None
build_dir = self.root
extra_scripts = []
self.get_optional('nodejs_scripts', extra_scripts.extend)
for target in extra_scripts:
nodejs_command = ['nodejs']
self.append_optional('nodejs_args', nodejs_command)
nodejs_command.append(target)
logging.debug('Running make with: %s', ' '.join(nodejs_command))
with output_header('NPM'):
child = subprocess.Popen(
nodejs_command,
cwd=build_dir,
)
_ = child.communicate()[0]
ret_code = child.returncode
if ret_code != 0:
debug_red('Running npm script %s failed', target)
return None
return 0
def _add_installer(self, installer, file):
assert installer is not None
assert file is not None
if not hasattr(installer, 'ExportedClass'):
debug_red('No exported class found in file %s', file)
return
if issubclass(installer.ExportedClass, BasicInstaller) is None:
debug_red(
'The exported class is not a subclass of BasicInstaller.')
return
if installer.ExportedClass.name is None:
debug_red('The exported class does not have proper name.')
return
InstallerClass = installer.ExportedClass # pylint: disable=C0103
if InstallerClass.name in self.installers.keys():
debug_red(
'Installer with the name \'%s\' already exists',
InstallerClass.name,
)
debug_cyan('Loading installer: \'%s\'', InstallerClass.name)
self.installers[InstallerClass.name] = InstallerClass
def _download_curl(self, name, package, root): # pylint: disable=R0201
logging.info('Trying to fetch with curl.')
if 'curl' not in package.keys() or not isinstance(
package['curl'], dict):
debug_red('Invalid curl node for packag %s.', name)
return None
if 'url' not in package['curl'].keys():
debug_red('The git node of %s does not have urlf field.', name)
return None
curl_node = package['curl']
url = curl_node['url']
path = root
file_name = file_name = url.split('/')[-1]
with suppress(OSError):
os.makedirs(path)
cmd = []
cmd.append(self.CURL_COMMAND)
if 'args' in curl_node.keys() and isinstance(curl_node['args'], str):
cmd.append(curl_node['args'])
cmd.append(url)
if 'output' in curl_node.keys() and isinstance(curl_node['output'],
str):
cmd.append('-o')
cmd.append(curl_node['output'])
file_name = curl_node['output']
else:
cmd.append('-O')
logging.debug('Fetching with curl and command: %s', cmd)
if execute_sanitized('curl', cmd, path) is None:
return None
file_path = os.path.join(path, file_name)
for ext in self.archive_extensions:
if file_path.endswith(ext):
self.extract_queue.append((file_path, name))
break
return 0
def checkout(self, git_node, path):
if 'checkout' in git_node.keys() and isinstance(
git_node['checkout'], str):
cmd = []
cmd.append(self.GIT_COMMAND)
cmd.append('checkout')
cmd.append(git_node['checkout'])
logging.debug(
'Checking out a git repository with "%s" ',
' '.join(cmd),
)
res = subprocess.run(cmd, cwd=path)
if res.returncode != 0:
debug_red('The checking out failed!')
return None
return 0
def execute(self, name):
assert name is not None
bash_lines = self.node['bash_lines']
in_bashrc = self.node.get('in_bashrc', False)
if isinstance(bash_lines, str):
bash_lines = [sanitize_input_variable(bash_lines)]
else:
bash_lines = map(sanitize_input_variable, bash_lines)
target_file = '~/.bashrc' if in_bashrc else os.path.join(
self.code_dir,
'setenv.sh',
)
target_file = sanitize_input_variable(target_file)
with open(target_file) as target:
target_file_lines = target.readlines()
with open(target_file, 'a') as target:
for line in bash_lines:
if not BashrcInstaller._check_line(line):
debug_red(
'Skipping bash line.\
The line is invalid: %s',
line,
)
continue
if line in target_file_lines:
debug_red(
'Skipping bash line.\
The line is already in the file: %s',
line,
)
continue
logging.debug('Writing bashrc line in %s', target_file)
target.write(line)
return 0
def execute(self, name):
assert name is not None
setup_file = os.path.join(self.root, 'setup.py')
if not os.path.isfile(setup_file):
debug_red(
'There isn\'t a setup.py file at the root of the package %s.',
name,
)
return None
setup_command = ['python']
setup_command.append('setup.py')
self.append_optional('setup_args', setup_command)
setup_command.append('install')
if self.node.get('prefix', False):
setup_command.append('--prefix')
setup_command.append(self.usr_dir)
self.append_optional('setup_install_args', setup_command)
logging.debug('Running setup.py with: %s', ' '.join(setup_command))
with output_header('Setup.py'):
child = subprocess.Popen(
setup_command,
cwd=self.root,
)
_ = child.communicate()[0]
ret_code = child.returncode
if ret_code != 0:
debug_red('Running setup.py failed')
return None
return 0
def execute(self, name):
assert name is not None
paths = self.node['path']
in_bashrc = self.node.get('in_bashrc', False)
if isinstance(paths, str):
paths = [paths]
paths = map(
lambda p: 'export LD_PATH=${{LD_PATH}}:{}'.format(
sanitize_input_variable(p), ),
paths,
)
target_file = '~/.bashrc' if in_bashrc else os.path.join(
self.code_dir,
'setenv.sh',
)
target_file = sanitize_input_variable(target_file)
with open(target_file) as target:
target_file_lines = target.read().splitlines()
with open(target_file, 'a') as target:
for line in list(paths):
if line in target_file_lines:
debug_red(
'Skipping library path export line.\
The line is already in the file: %s',
line,
)
continue
logging.debug('Adding library export in %s', target_file)
target.write(line)
target.write('\n')
return 0
def execute_sanitized(name, command, cwd, supress_output=False):
assert name is not None
assert command is not None
assert cwd is not None
command = list(map(sanitize_input_variable, command))
stdout = subprocess.PIPE if supress_output else None
with output_header(name):
# TODO: This is very dangerous wiht shell=True
# Think of something better at some point
command = ' '.join(command) if isinstance(command, list) else command
child = subprocess.Popen(
command,
cwd=cwd,
stdout=stdout,
shell=True,
)
_ = child.communicate()[0]
ret_code = child.returncode
if ret_code != 0:
debug_red('Running %s failed', name)
return None
return 0
def get_output(name, command, cwd, supress_output=False):
assert name is not None
assert command is not None
assert cwd is not None
command = list(map(sanitize_input_variable, command))
with output_header(name):
child = subprocess.Popen(
command,
cwd=cwd,
stdout=subprocess.PIPE,
)
for line in child.stdout:
if not supress_output:
print(line)
yield line
_ = child.communicate()[0]
ret_code = child.returncode
if ret_code != 0:
debug_red('Running %s failed', name)
return None
return 0
def execute(self, name):
assert name is not None
make_command = ['make']
self.append_optional('make_args', make_command)
build_dir = os.path.join(self.root, 'build/')
if not os.path.isdir(build_dir):
build_dir = self.root
if not os.path.isdir(build_dir):
debug_red('There is no build directory. Connot run make')
return None
logging.debug('Running make with: %s', ' '.join(make_command))
logging.debug('Build directory: %s', build_dir)
with output_header('Make'):
child = subprocess.Popen(
make_command,
cwd=build_dir,
)
_ = child.communicate()[0]
ret_code = child.returncode
if ret_code != 0:
debug_red('Running make failed')
return None
extra_targets = []
self.get_optional('make_extra_targets', extra_targets.extend)
for target in extra_targets:
make_command = ['make']
self.append_optional('make_args', make_command)
make_command.append(target)
logging.debug('Running make with: %s', ' '.join(make_command))
with output_header('Make'):
child = subprocess.Popen(
make_command,
cwd=build_dir,
)
_ = child.communicate()[0]
ret_code = child.returncode
if ret_code != 0:
debug_red('Running make target %s failed', target)
return None
return 0
def _add_command(self, command, file):
assert command is not None
assert file is not None
if not hasattr(command, 'ExportedClass'):
debug_red('No exported command class found in file %s', file)
return
if command.ExportedClass.name is None:
debug_red('The exported class does not have proper name.')
return
CommandClass = command.ExportedClass # pylint: disable=C0103
if CommandClass.name in self.commands.keys():
debug_red(
'Command with the name \'%s\' already exists',
CommandClass.name,
)
debug_cyan('Loading command: \'%s\'', CommandClass.name)
self.commands[CommandClass.name] = CommandClass
def _download_git(self, name, package, root): # pylint: disable=R0201,R0911
assert name is not None
assert root is not None
assert package is not None
logging.info('Trying to fetch with git.')
if 'git' not in package.keys() or not isinstance(package['git'], dict):
debug_red('Invalid git node for packag %s.', name)
return None
if 'url' not in package['git'].keys():
debug_red('The git node of %s does not have urlf field.', name)
return None
git_node = package['git']
url = git_node['url']
# TODO: Do not join the root here but pass it from above
path = root
cmd = []
cmd.extend([self.GIT_COMMAND, 'clone'])
if 'args' in git_node.keys() and isinstance(git_node['args'], str):
cmd.append(git_node['args'])
if self.git_ssh:
logging.debug('Trying to use ssh with git')
match = self.GIT_HTTPS_RE.match(url)
if match:
url = 'git@' + match.group(1) + ':' + match.group(2)
cmd.append(url)
elif self.GIT_SSH_RE.match(url) is not None:
cmd.append(url)
else:
debug_red('Bad git url for %s: %s', name, url)
return None
else:
logging.debug('Trying to use https with git')
match = self.GIT_SSH_RE.match(url)
if match:
url = 'https://' + match.group(1) + '/' + match.group(2)
cmd.append(url)
elif self.GIT_HTTPS_RE.match(url) is not None:
cmd.append(url)
else:
debug_red('Bad git url for %s: %s', name, url)
return None
if os.path.exists(path):
debug_red('The given path already exists: %s', path)
return None
cmd.append(path)
logging.debug('Fetching with git and command: %s', cmd)
if subprocess.call(cmd) != 0:
debug_red('The fetching failed!')
return None
# TODO: Mark somehow that the contents have been cloned but the step is not fully complete
if self.checkout(git_node, path) is None:
return None
return 0
