def _install_into_venv(self, instance, requirements,
upgrade=False, extra_env_overrides=None):
venv_dir = self._venv_directory_for(instance)
base_pip = [sh.joinpths(venv_dir, 'bin', 'pip')]
env_overrides = {
'PATH': os.pathsep.join([sh.joinpths(venv_dir, "bin"),
env.get_key('PATH', default_value='')]),
'VIRTUAL_ENV': venv_dir,
}
if extra_env_overrides:
env_overrides.update(extra_env_overrides)
cmd = list(base_pip) + ['install']
if upgrade:
cmd.append("--upgrade")
if isinstance(requirements, six.string_types):
cmd.extend([
'--requirement',
requirements
])
else:
for req in requirements:
cmd.append(str(req))
count = self.install_counters.get(instance.name, 0)
self.install_counters[instance.name] = count + 1
out_filename = sh.joinpths(self.log_dir, "venv-install-%s-%s.log" % (instance.name, count))
sh.execute_save_output(cmd, out_filename, env_overrides=env_overrides)
def _yyoom(self, arglist, on_completed=None):
if not on_completed:
on_completed = lambda data, errored: None
if not sh.isdir(self._logs_dir):
sh.mkdirslist(self._logs_dir)
with tempfile.NamedTemporaryFile(suffix=".json") as fh:
cmdline = [
self.yyoom_executable,
"--output-file", fh.name,
"--verbose",
]
cmdline.extend(arglist)
log_filename = sh.joinpths(self._logs_dir,
_generate_log_filename(arglist))
LOG.debug("Running yyoom: log output will be placed in %s",
log_filename)
try:
sh.execute_save_output(cmdline, log_filename)
except excp.ProcessExecutionError:
with excp.reraise():
try:
fh.seek(0)
data = utils.parse_json(fh.read())
except Exception:
LOG.exception("Failed to parse YYOOM output")
else:
on_completed(data, True)
else:
fh.seek(0)
data = utils.parse_json(fh.read())
on_completed(data, False)
return data
def download_dependencies(download_dir, pips_to_download, output_filename):
if not pips_to_download:
return
# NOTE(aababilov): pip has issues with already downloaded files
if sh.isdir(download_dir):
for filename in sh.listdir(download_dir, files_only=True):
sh.unlink(filename)
else:
sh.mkdir(download_dir)
# Clean out any previous paths that we don't want around.
build_path = sh.joinpths(download_dir, ".build")
if sh.isdir(build_path):
sh.deldir(build_path)
sh.mkdir(build_path)
cmdline = [
PIP_EXECUTABLE,
'-v',
'install',
'-I',
'-U',
'--download',
download_dir,
'--build',
build_path,
# Don't download wheels since we lack the ability to create
# rpms from them (until future when we will have it, if ever)...
"--no-use-wheel",
]
for p in pips_to_download:
for p_seg in _split(p):
if p_seg:
cmdline.append(p_seg)
sh.execute_save_output(cmdline, output_filename)
def _build_dependencies(self):
package_files = self.download_dependencies()
def filter_files(package_files):
for p in package_files:
banned = False
for k in self.BANNED_PACKAGES:
if k in p.lower():
banned = True
if banned:
continue
yield p
package_files = [f for f in filter_files(package_files)]
if not package_files:
LOG.info("No RPM packages of OpenStack dependencies to build")
return
package_base_names = [sh.basename(f) for f in package_files]
utils.log_iterable(sorted(package_base_names),
logger=LOG,
header=("Building %s dependency RPM"
" packages") % (len(package_files)))
with utils.progress_bar(name='Building',
max_am=len(package_files)) as p_bar:
for (i, filename) in enumerate(sorted(package_files)):
cmdline = self.py2rpm_start_cmdline() + ["--", filename]
build_filename = "py2rpm-%s.out" % sh.basename(filename)
out_filename = sh.joinpths(self.log_dir, build_filename)
sh.execute_save_output(cmdline,
out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
def download_dependencies(download_dir, pips_to_download, output_filename):
if not pips_to_download:
return
# NOTE(aababilov): pip has issues with already downloaded files
if sh.isdir(download_dir):
for filename in sh.listdir(download_dir, files_only=True):
sh.unlink(filename)
else:
sh.mkdir(download_dir)
# Clean out any previous paths that we don't want around.
build_path = sh.joinpths(download_dir, ".build")
if sh.isdir(build_path):
sh.deldir(build_path)
sh.mkdir(build_path)
# Ensure certain directories exist that we want to exist (but we don't
# want to delete them run after run).
cache_path = sh.joinpths(download_dir, ".cache")
if not sh.isdir(cache_path):
sh.mkdir(cache_path)
cmdline = [
PIP_EXECUTABLE, '-v',
'install', '-I', '-U',
'--download', download_dir,
'--build', build_path,
'--download-cache', cache_path,
]
# Don't download wheels...
#
# See: https://github.com/pypa/pip/issues/1439
if dist_version.StrictVersion(PIP_VERSION) >= dist_version.StrictVersion('1.5'):
cmdline.append("--no-use-wheel")
cmdline.extend([str(p) for p in pips_to_download])
sh.execute_save_output(cmdline, output_filename)
def _install_into_venv(self,
instance,
requirements,
upgrade=False,
extra_env_overrides=None):
venv_dir = self._venv_directory_for(instance)
base_pip = [sh.joinpths(venv_dir, 'bin', 'pip')]
env_overrides = {
'PATH':
os.pathsep.join([
sh.joinpths(venv_dir, "bin"),
env.get_key('PATH', default_value='')
]),
'VIRTUAL_ENV':
venv_dir,
}
if extra_env_overrides:
env_overrides.update(extra_env_overrides)
cmd = list(base_pip) + ['install']
if upgrade:
cmd.append("--upgrade")
if isinstance(requirements, six.string_types):
cmd.extend(['--requirement', requirements])
else:
for req in requirements:
cmd.append(str(req))
count = self.install_counters.get(instance.name, 0)
self.install_counters[instance.name] = count + 1
out_filename = sh.joinpths(
self.log_dir, "venv-install-%s-%s.log" % (instance.name, count))
sh.execute_save_output(cmd, out_filename, env_overrides=env_overrides)
def _yyoom(self, arglist, on_completed=None):
if not on_completed:
on_completed = lambda data, errored: None
if not sh.isdir(self._logs_dir):
sh.mkdirslist(self._logs_dir)
with tempfile.NamedTemporaryFile(suffix=".json") as fh:
cmdline = [
self.yyoom_executable,
"--output-file",
fh.name,
"--verbose",
]
cmdline.extend(arglist)
log_filename = sh.joinpths(self._logs_dir,
_generate_log_filename(arglist))
LOG.debug("Running yyoom: log output will be placed in %s",
log_filename)
try:
sh.execute_save_output(cmdline, log_filename)
except excp.ProcessExecutionError:
with excp.reraise():
try:
fh.seek(0)
data = utils.parse_json(fh.read())
except Exception:
LOG.exception("Failed to parse YYOOM output")
else:
on_completed(data, True)
else:
fh.seek(0)
data = utils.parse_json(fh.read())
on_completed(data, False)
return data
def _execute_make(self, filename, marks_dir):
cmdline = ["make", "-f", filename, "-j", str(self._jobs)]
out_filename = sh.joinpths(self.log_dir,
"%s.log" % sh.basename(filename))
sh.execute_save_output(cmdline,
cwd=marks_dir,
out_filename=out_filename)
def _build_from_app_dir(self, instance):
app_dir = instance.get_option('app_dir')
cmdline = self.py2rpm_start_cmdline()
cmdline.extend(["--source-only", "--", app_dir])
out_filename = sh.joinpths(self.log_dir, "py2rpm-build-%s.log" % (instance.name))
sh.execute_save_output(cmdline, cwd=app_dir, out_filename=out_filename,
quiet=True)
def _build_dependencies(self):
package_files = self.download_dependencies()
def filter_files(package_files):
for p in package_files:
banned = False
for k in self.BANNED_PACKAGES:
if k in p.lower():
banned = True
if banned:
continue
yield p
package_files = [f for f in filter_files(package_files)]
if not package_files:
LOG.info("No RPM packages of OpenStack dependencies to build")
return
package_base_names = [sh.basename(f) for f in package_files]
utils.log_iterable(sorted(package_base_names), logger=LOG,
header=("Building %s dependency RPM"
" packages") % (len(package_files)))
with utils.progress_bar(name='Building',
max_am=len(package_files)) as p_bar:
for (i, filename) in enumerate(sorted(package_files)):
cmdline = self.py2rpm_start_cmdline() + ["--", filename]
build_filename = "py2rpm-%s.out" % sh.basename(filename)
out_filename = sh.joinpths(self.log_dir, build_filename)
sh.execute_save_output(cmdline, out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
def _build_from_spec(self, instance, spec_filename, patches=None):
pkg_dir = instance.get_option('app_dir')
if sh.isfile(sh.joinpths(pkg_dir, "setup.py")):
self._write_python_tarball(instance, pkg_dir, ENSURE_NOT_MISSING)
else:
self._write_git_tarball(instance, pkg_dir, spec_filename)
self._copy_sources(instance)
if patches:
self._copy_patches(patches)
cmdline = [self.specprint_executable]
cmdline.extend(['-f', spec_filename])
spec_details = json.loads(sh.execute(cmdline)[0])
rpm_requires = []
for k in ('requires', 'requirenevrs'):
try:
rpm_requires.extend(spec_details['headers'][k])
except (KeyError, TypeError):
pass
if rpm_requires:
buff = six.StringIO()
buff.write("# %s\n" % instance.name)
if rpm_requires:
for req in rpm_requires:
buff.write("%s\n" % req)
buff.write("\n")
sh.append_file(self.rpm_build_requires_filename, buff.getvalue())
self._copy_startup_scripts(instance, spec_details)
cmdline = [
self.rpmbuild_executable,
"-bs",
"--define", "_topdir %s" % self.rpmbuild_dir,
spec_filename,
]
out_filename = sh.joinpths(self.log_dir, "rpmbuild-%s.log" % instance.name)
sh.execute_save_output(cmdline, out_filename)
def download_dependencies(download_dir, pips_to_download, output_filename):
if not pips_to_download:
return
# NOTE(aababilov): pip has issues with already downloaded files
if sh.isdir(download_dir):
for filename in sh.listdir(download_dir, files_only=True):
sh.unlink(filename)
else:
sh.mkdir(download_dir)
# Clean out any previous paths that we don't want around.
build_path = sh.joinpths(download_dir, ".build")
if sh.isdir(build_path):
sh.deldir(build_path)
sh.mkdir(build_path)
cmdline = [
PIP_EXECUTABLE, '-v',
'install', '-I', '-U',
'--download', download_dir,
'--build', build_path,
# Don't download wheels since we lack the ability to create
# rpms from them (until future when we will have it, if ever)...
"--no-use-wheel",
]
for p in pips_to_download:
for p_seg in _split(p):
if p_seg:
cmdline.append(p_seg)
sh.execute_save_output(cmdline, output_filename)
def build_srpm(self, source, log_filename, release=None, with_tests=False):
cmdline = self._start_cmdline() + ["--source-only"]
if release is not None:
cmdline.extend(["--release", release])
if with_tests:
cmdline.append("--with-tests")
cmdline.extend(["--", source])
out_filename = sh.joinpths(self._log_dir, "py2rpm-build-%s.log" % log_filename)
sh.execute_save_output(cmdline, cwd=source, out_filename=out_filename)
def _build_from_app_dir(self, instance, params):
app_dir = instance.get_option('app_dir')
cmdline = self._py2rpm_start_cmdline()
cmdline.extend(["--source-only"])
if 'release' in params:
cmdline.extend(["--release", params["release"]])
cmdline.extend(["--", app_dir])
out_filename = sh.joinpths(self.log_dir, "py2rpm-build-%s.log" % (instance.name))
sh.execute_save_output(cmdline, cwd=app_dir, out_filename=out_filename)
def build_srpm(self, source, log_filename, release=None, with_tests=False):
cmdline = self._start_cmdline() + ["--source-only", "--debug"]
if release is not None:
cmdline.extend(["--release", release])
if with_tests:
cmdline.append("--with-tests")
cmdline.extend(["--", source])
out_filename = sh.joinpths(self._log_dir,
"py2rpm-build-%s.log" % log_filename)
sh.execute_save_output(cmdline, out_filename, cwd=source)
def _build_from_app_dir(self, instance, params):
app_dir = instance.get_option('app_dir')
cmdline = self._py2rpm_start_cmdline()
cmdline.extend(["--source-only"])
if 'release' in params:
cmdline.extend(["--release", params["release"]])
cmdline.extend(["--", app_dir])
out_filename = sh.joinpths(self.log_dir,
"py2rpm-build-%s.log" % (instance.name))
sh.execute_save_output(cmdline, cwd=app_dir, out_filename=out_filename)
def _try_download_dependencies(self, attempt, pips_to_download, pip_download_dir):
cmdline = [
self.pipdownload_executable,
'-d', pip_download_dir,
'-v',
]
cmdline.extend(sorted([str(p) for p in pips_to_download]))
out_filename = sh.joinpths(self.log_dir,
"pip-download-attempt-%s.log" % (attempt))
sh.execute_save_output(cmdline, out_filename=out_filename)
def _try_download_dependencies(self, attempt, pips_to_download,
pip_download_dir):
cmdline = [
self.pipdownload_executable,
'-d',
pip_download_dir,
'-v',
]
cmdline.extend(sorted([str(p) for p in pips_to_download]))
out_filename = sh.joinpths(self.log_dir,
"pip-download-attempt-%s.log" % (attempt))
sh.execute_save_output(cmdline, out_filename=out_filename)
def _try_download_dependencies(self, attempt, pips_to_download,
pip_download_dir,
pip_cache_dir,
pip_build_dir):
pips_to_download = [str(p) for p in pips_to_download]
cmdline = [
self.pip_executable,
"install",
"--download", pip_download_dir,
"--download-cache", pip_cache_dir,
"--build", pip_build_dir,
]
cmdline.extend(sorted(pips_to_download))
download_filename = "pip-download-attempt-%s.log"
download_filename = download_filename % (attempt)
out_filename = sh.joinpths(self.log_dir, download_filename)
sh.execute_save_output(cmdline, out_filename=out_filename)
def package_start(self):
super(VenvDependencyHandler, self).package_start()
self.install_counters.clear()
base_cmd = env.get_key('VENV_CMD', default_value='virtualenv')
for instance in self.instances:
if not self._is_buildable(instance):
continue
# Create a virtualenv...
venv_dir = self._venv_directory_for(instance)
sh.mkdirslist(venv_dir, tracewriter=self.tracewriter)
cmd = [base_cmd, '--clear', venv_dir]
LOG.info("Creating virtualenv at %s", colorizer.quote(venv_dir))
out_filename = sh.joinpths(self.log_dir, "venv-create-%s.log" % (instance.name))
sh.execute_save_output(cmd, out_filename)
self._install_into_venv(instance,
self.PREREQUISITE_UPGRADE_PKGS,
upgrade=True)
def _write_python_tarball(self, instance, pkg_dir, ensure_exists=None):
def prefix_exists(text, in_what):
for t in in_what:
if t.startswith(text):
return True
return False
pkg_name = instance.egg_info['name']
version = instance.egg_info['version']
base_name = "%s-%s" % (pkg_name, version)
cmdline = [
sys.executable,
"setup.py",
"sdist",
"--formats=tar",
"--dist-dir",
self.rpm_sources_dir,
]
out_filename = sh.joinpths(self.log_dir,
"sdist-%s.log" % (instance.name))
sh.execute_save_output(cmdline, cwd=pkg_dir, out_filename=out_filename)
archive_name = sh.joinpths(self.rpm_sources_dir,
"%s.tar" % (base_name))
if ensure_exists:
with contextlib.closing(tarfile.open(archive_name, 'r')) as tfh:
tar_entries = [t.path for t in tfh.getmembers()]
missing_paths = {}
for path in ensure_exists:
tar_path = sh.joinpths(base_name, path)
source_path = sh.joinpths(pkg_dir, path)
if not prefix_exists(tar_path,
tar_entries) and sh.exists(source_path):
missing_paths[tar_path] = source_path
if missing_paths:
utils.log_iterable(
sorted(missing_paths.keys()),
logger=LOG,
header='%s paths were not archived and will now be' %
(len(missing_paths)))
with contextlib.closing(tarfile.open(archive_name,
'a')) as tfh:
for (tar_path, source_path) in missing_paths.items():
tfh.add(source_path, tar_path)
sh.gzip(archive_name)
sh.unlink(archive_name)
def package_start(self):
super(VenvDependencyHandler, self).package_start()
self.install_counters.clear()
base_cmd = env.get_key('VENV_CMD', default_value='virtualenv')
for instance in self.instances:
if not self._is_buildable(instance):
continue
# Create a virtualenv...
venv_dir = self._venv_directory_for(instance)
sh.mkdirslist(venv_dir, tracewriter=self.tracewriter)
cmd = [base_cmd, '--clear', venv_dir]
LOG.info("Creating virtualenv at %s", colorizer.quote(venv_dir))
out_filename = sh.joinpths(self.log_dir,
"venv-create-%s.log" % (instance.name))
sh.execute_save_output(cmd, out_filename)
self._install_into_venv(instance,
self.PREREQUISITE_UPGRADE_PKGS,
upgrade=True)
def _build_from_spec(self, instance, spec_filename, patches=None):
pkg_dir = instance.get_option('app_dir')
if sh.isfile(sh.joinpths(pkg_dir, "setup.py")):
self._write_python_tarball(instance, pkg_dir, ENSURE_NOT_MISSING)
else:
self._write_git_tarball(instance, pkg_dir, spec_filename)
self._copy_sources(instance)
if patches:
self._copy_patches(patches)
self._copy_startup_scripts(instance, spec_filename)
cmdline = [
self.rpmbuild_executable,
"-bs",
"--define", "_topdir %s" % self.rpmbuild_dir,
spec_filename,
]
out_filename = sh.joinpths(self.log_dir, "rpmbuild-%s.log" % instance.name)
sh.execute_save_output(cmdline, out_filename=out_filename)
def _build_openstack(self):
if not self.package_dirs:
LOG.warn("No RPM packages of OpenStack installs to build")
return
component_names = [self._get_component_name(d)
for d in self.package_dirs]
utils.log_iterable(sorted(component_names), logger=LOG,
header=("Building %s OpenStack RPM"
" packages") % (len(self.package_dirs)))
with utils.progress_bar(name='Building',
max_am=len(self.package_dirs)) as p_bar:
for (i, pkg_dir) in enumerate(sorted(self.package_dirs)):
component_name = self._get_component_name(pkg_dir)
cmdline = self.py2rpm_start_cmdline() + ["--", pkg_dir]
out_filename = sh.joinpths(self.log_dir,
"py2rpm.%s.out" % (component_name))
sh.execute_save_output(cmdline, out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
def _write_python_tarball(self, instance, pkg_dir, ensure_exists=None):
def prefix_exists(text, in_what):
for t in in_what:
if t.startswith(text):
return True
return False
pkg_name = instance.egg_info['name']
version = instance.egg_info['version']
base_name = "%s-%s" % (pkg_name, version)
cmdline = [
sys.executable,
"setup.py",
"sdist",
"--formats=tar",
"--dist-dir", self.rpm_sources_dir,
]
env_overrides = {
'PBR_VERSION': version,
}
out_filename = sh.joinpths(self.log_dir, "sdist-%s.log" % (instance.name))
sh.execute_save_output(cmdline, out_filename,
cwd=pkg_dir, env_overrides=env_overrides)
archive_name = sh.joinpths(self.rpm_sources_dir, "%s.tar" % (base_name))
if ensure_exists:
with contextlib.closing(tarfile.open(archive_name, 'r')) as tfh:
tar_entries = [t.path for t in tfh.getmembers()]
missing_paths = {}
for path in ensure_exists:
tar_path = sh.joinpths(base_name, path)
source_path = sh.joinpths(pkg_dir, path)
if not prefix_exists(tar_path, tar_entries) and sh.exists(source_path):
missing_paths[tar_path] = source_path
if missing_paths:
utils.log_iterable(sorted(missing_paths.keys()),
logger=LOG,
header='%s paths were not archived and will now be' % (len(missing_paths)))
with contextlib.closing(tarfile.open(archive_name, 'a')) as tfh:
for (tar_path, source_path) in missing_paths.items():
tfh.add(source_path, tar_path)
sh.gzip(archive_name)
sh.unlink(archive_name)
def _try_download_dependencies(self, attempt, pips_to_download,
pip_download_dir, pip_cache_dir,
pip_build_dir):
pips_to_download = [str(p) for p in pips_to_download]
cmdline = [
self.pip_executable,
"install",
"--download",
pip_download_dir,
"--download-cache",
pip_cache_dir,
"--build",
pip_build_dir,
]
cmdline.extend(sorted(pips_to_download))
download_filename = "pip-download-attempt-%s.out"
download_filename = download_filename % (attempt)
out_filename = sh.joinpths(self.log_dir, download_filename)
sh.execute_save_output(cmdline, out_filename=out_filename)
def _build_from_spec(self, instance, spec_filename, patches=None):
pkg_dir = instance.get_option('app_dir')
if sh.isfile(sh.joinpths(pkg_dir, "setup.py")):
self._write_python_tarball(instance, pkg_dir, ENSURE_NOT_MISSING)
else:
self._write_git_tarball(instance, pkg_dir, spec_filename)
self._copy_sources(instance)
if patches:
self._copy_patches(patches)
self._copy_startup_scripts(instance, spec_filename)
cmdline = [
self.rpmbuild_executable,
"-bs",
"--define",
"_topdir %s" % self.rpmbuild_dir,
spec_filename,
]
out_filename = sh.joinpths(self.log_dir,
"rpmbuild-%s.log" % instance.name)
sh.execute_save_output(cmdline, out_filename=out_filename)
def _build_openstack(self):
if not self.package_dirs:
LOG.warn("No RPM packages of OpenStack installs to build")
return
component_names = [
self._get_component_name(d) for d in self.package_dirs
]
utils.log_iterable(sorted(component_names),
logger=LOG,
header=("Building %s OpenStack RPM"
" packages") % (len(self.package_dirs)))
with utils.progress_bar(name='Building',
max_am=len(self.package_dirs)) as p_bar:
for (i, pkg_dir) in enumerate(sorted(self.package_dirs)):
component_name = self._get_component_name(pkg_dir)
cmdline = self.py2rpm_start_cmdline() + ["--", pkg_dir]
out_filename = sh.joinpths(self.log_dir,
"py2rpm.%s.out" % (component_name))
sh.execute_save_output(cmdline,
out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
def _write_git_tarball(self, instance, pkg_dir, spec_filename):
cmdline = [
"rpm", "-q", "--specfile", spec_filename, "--qf",
"%{NAME}-%{VERSION}\n"
]
tar_base = sh.execute(cmdline, cwd=pkg_dir)[0].splitlines()[0].strip()
# NOTE(harlowja): git 1.7.1 from RHEL doesn't understand --format=tar.gz
output_filename = sh.joinpths(self.rpm_sources_dir,
"%s.tar" % tar_base)
cmdline = [
"git",
"archive",
"--format=tar",
"--prefix=%s/" % tar_base,
"--output=%s" % output_filename,
"HEAD",
]
out_filename = sh.joinpths(self.log_dir,
"git-tar-%s.log" % instance.name)
sh.execute_save_output(cmdline, cwd=pkg_dir, out_filename=out_filename)
sh.gzip(output_filename)
sh.unlink(output_filename)
def _write_git_tarball(self, instance, pkg_dir, spec_filename):
cmdline = [
"rpm",
"-q",
"--specfile", spec_filename,
"--qf", "%{NAME}-%{VERSION}\n"
]
tar_base = sh.execute(cmdline, cwd=pkg_dir)[0].splitlines()[0].strip()
# NOTE(harlowja): git 1.7.1 from RHEL doesn't understand --format=tar.gz
output_filename = sh.joinpths(self.rpm_sources_dir, "%s.tar" % tar_base)
cmdline = [
"git",
"archive",
"--format=tar",
"--prefix=%s/" % tar_base,
"--output=%s" % output_filename,
"HEAD",
]
out_filename = sh.joinpths(self.log_dir, "git-tar-%s.log" % instance.name)
sh.execute_save_output(cmdline, cwd=pkg_dir, out_filename=out_filename)
sh.gzip(output_filename)
sh.unlink(output_filename)
def _execute_make(self, filename, marks_dir, jobs):
cmdline = ["make", "-f", filename, "-j", str(jobs)]
out_filename = sh.joinpths(self._log_dir, "%s.log" % sh.basename(filename))
sh.execute_save_output(cmdline, out_filename, cwd=marks_dir)
