def get_directory_details(path):
if not sh.isdir(path):
raise IOError("Can not detail non-existent directory %s" % (path))
# Check if we already got the details of this dir previously
path = sh.abspth(path)
cache_key = "d:%s" % (sh.abspth(path))
if cache_key in EGGS_DETAILED:
return EGGS_DETAILED[cache_key]
req = extract(path)
req.source_dir = path
req.run_egg_info()
dependencies = []
for d in req.requirements():
if not d.startswith("-e") and d.find("#"):
d = d.split("#")[0]
d = d.strip()
if d:
dependencies.append(d)
details = {
'req': req.req,
'dependencies': dependencies,
'name': req.name,
'pkg_info': req.pkg_info(),
'dependency_links': req.dependency_links,
'version': req.installed_version,
}
EGGS_DETAILED[cache_key] = details
return details
def get_directory_details(path):
if not sh.isdir(path):
raise IOError("Can not detail non-existent directory %s" % (path))
# Check if we already got the details of this dir previously
path = sh.abspth(path)
cache_key = "d:%s" % (sh.abspth(path))
if cache_key in EGGS_DETAILED:
return EGGS_DETAILED[cache_key]
req = extract(path)
req.source_dir = path
req.run_egg_info()
dependencies = []
for d in req.requirements():
if not d.startswith("-e") and d.find("#"):
d = d.split("#")[0]
d = d.strip()
if d:
dependencies.append(d)
details = {
'req': req.req,
'dependencies': dependencies,
'name': req.name,
'pkg_info': req.pkg_info(),
'dependency_links': req.dependency_links,
'version': req.installed_version,
}
EGGS_DETAILED[cache_key] = details
return details
def load_all(cls, path=settings.DISTRO_DIR):
"""Returns a list of the known distros."""
results = []
input_files = glob.glob(sh.joinpths(path, '*.yaml'))
if not input_files:
raise RuntimeError(
'Did not find any distro definition files in %r' %
path)
for fn in input_files:
cls_kvs = None
filename = sh.abspth(fn)
LOG.audit("Attempting to load distro definition from %r" % (filename))
try:
with open(filename, 'r') as f:
cls_kvs = yaml.load(f)
except (IOError, yaml.YAMLError) as err:
LOG.warning('Could not load distro definition from %r: %s',
filename, err)
if cls_kvs is not None:
try:
results.append(cls(**cls_kvs))
except Exception as err:
LOG.warning('Could not initialize instance %s using parameter map %s: %s',
cls, cls_kvs, err)
return results
def package_finish(self):
super(VenvDependencyHandler, self).package_finish()
for instance in self.instances:
if not self._is_buildable(instance):
continue
venv_dir = sh.abspth(self._venv_directory_for(instance))
# Replace paths with virtualenv deployment directory.
if self.opts.get('venv_deploy_dir'):
deploy_dir = sh.joinpths(self.opts.get('venv_deploy_dir'),
instance.name)
replacer = functools.partial(
re.subn, re.escape(instance.get_option('component_dir')),
deploy_dir)
bin_dir = sh.joinpths(venv_dir, 'bin')
adjustments, files_replaced = self._replace_deployment_paths(bin_dir,
replacer)
if files_replaced:
LOG.info("Adjusted %s deployment path(s) in %s files",
adjustments, files_replaced)
# Create a tarball containing the virtualenv.
tar_filename = sh.joinpths(venv_dir, '%s-venv.tar.gz' % instance.name)
LOG.info("Making tarball of %s built for %s at %s", venv_dir,
instance.name, tar_filename)
with contextlib.closing(tarfile.open(tar_filename, "w:gz")) as tfh:
for path in sh.listdir(venv_dir, recursive=True):
tfh.add(path, recursive=False, arcname=path[len(venv_dir):])
def build_all_srpms(self, package_files, tracewriter, jobs):
(_fn,
content) = utils.load_template(sh.joinpths("packaging", "makefiles"),
"source.mk")
scripts_dir = sh.abspth(
sh.joinpths(settings.TEMPLATE_DIR, "packaging", "scripts"))
cmdline = self._start_cmdline(escape_values=True)[1:] + [
"--scripts-dir",
scripts_dir,
"--source-only",
"--rpm-base",
self._rpmbuild_dir,
"--debug",
]
executable = " ".join(self._start_cmdline()[0:1])
params = {
"DOWNLOADS_DIR": self._download_dir,
"LOGS_DIR": self._log_dir,
"PY2RPM": executable,
"PY2RPM_FLAGS": " ".join(cmdline)
}
marks_dir = sh.joinpths(self._deps_dir, "marks-deps")
if not sh.isdir(marks_dir):
sh.mkdirslist(marks_dir, tracewriter=tracewriter)
makefile_path = sh.joinpths(self._deps_dir, "deps.mk")
sh.write_file(makefile_path,
utils.expand_template(content, params),
tracewriter=tracewriter)
utils.log_iterable(package_files,
header="Building %s SRPM packages using %s jobs" %
(len(package_files), jobs),
logger=LOG)
self._execute_make(makefile_path, marks_dir, jobs)
def build_all_srpms(self, package_files, tracewriter, jobs):
(_fn, content) = utils.load_template(sh.joinpths("packaging", "makefiles"), "source.mk")
scripts_dir = sh.abspth(sh.joinpths(settings.TEMPLATE_DIR, "packaging", "scripts"))
cmdline = self._start_cmdline(escape_values=True)[1:] + [
"--scripts-dir", scripts_dir,
"--source-only",
"--rpm-base", self._rpmbuild_dir
]
executable = " ".join(self._start_cmdline()[0:1])
params = {
"DOWNLOADS_DIR": self._download_dir,
"LOGS_DIR": self._log_dir,
"PY2RPM": executable,
"PY2RPM_FLAGS": " ".join(cmdline)
}
marks_dir = sh.joinpths(self._deps_dir, "marks-deps")
if not sh.isdir(marks_dir):
sh.mkdirslist(marks_dir, tracewriter=tracewriter)
makefile_path = sh.joinpths(self._deps_dir, "deps.mk")
sh.write_file(makefile_path, utils.expand_template(content, params),
tracewriter=tracewriter)
utils.log_iterable(package_files,
header="Building %s SRPM packages using %s jobs" %
(len(package_files), jobs),
logger=LOG)
self._execute_make(makefile_path, marks_dir, jobs)
def _uninstall_dirs(self):
dirs_made = self.tracereader.dirs_made()
dirs_alive = filter(sh.isdir, [sh.abspth(d) for d in dirs_made])
if dirs_alive:
utils.log_iterable(dirs_alive, logger=LOG,
header="Removing %s created directories" % (len(dirs_alive)))
for dir_name in dirs_alive:
sh.deldir(dir_name, run_as_root=True)
def _write_rc_file(self):
fn = sh.abspth(settings.gen_rc_filename('core'))
writer = env_rc.RcWriter(self.cfg, self.root_dir)
if not sh.isfile(fn):
LOG.info("Generating a file at %s that will contain your environment settings.", colorizer.quote(fn))
writer.write(fn)
else:
LOG.info("Updating a file at %s that contains your environment settings.", colorizer.quote(fn))
am_upd = writer.update(fn)
LOG.info("Updated %s settings.", colorizer.quote(am_upd))
def expand_patches(paths, patch_ext='.patch'):
if not paths:
return []
all_paths = []
# Expand patch files/dirs
for path in paths:
path = sh.abspth(path)
if sh.isdir(path):
all_paths.extend([p for p in sh.listdir(path, files_only=True)])
else:
all_paths.append(path)
# Now filter on valid patches
return [p for p in all_paths if _is_patch(p, patch_ext=patch_ext)]
def expand_patches(paths):
if not paths:
return []
all_paths = []
# Expand patch files/dirs
for path in paths:
path = sh.abspth(path)
if sh.isdir(path):
all_paths.extend([p for p in sh.listdir(path, files_only=True)])
else:
all_paths.append(path)
# Now filter on valid patches
return [p for p in all_paths if _is_patch(p)]
def read_requirement_files(files):
pip_requirements = []
session = pip_download.PipSession()
for filename in files:
if sh.isfile(filename):
cache_key = "f:%s:%s" % (sh.abspth(filename), sh.getsize(filename))
with REQUIREMENT_FILE_CACHE_LOCK:
try:
reqs = REQUIREMENT_FILE_CACHE[cache_key]
except KeyError:
reqs = tuple(
pip_req.parse_requirements(filename, session=session))
REQUIREMENT_FILE_CACHE[cache_key] = reqs
pip_requirements.extend(reqs)
return (pip_requirements, [req.req for req in pip_requirements])
def _patches(self):
your_patches = []
in_patches = self.get_option('patches', 'package')
if in_patches:
for path in in_patches:
path = sh.abspth(path)
if sh.isdir(path):
for c_path in sh.listdir(path, files_only=True):
tgt_fn = sh.joinpths(self.build_paths['sources'], sh.basename(c_path))
sh.copy(c_path, tgt_fn)
your_patches.append(sh.basename(tgt_fn))
else:
tgt_fn = sh.joinpths(self.build_paths['sources'], sh.basename(path))
sh.copy(path, tgt_fn)
your_patches.append(sh.basename(tgt_fn))
return your_patches
def read_requirement_files(files):
pip_requirements = []
session = pip_download.PipSession()
for filename in files:
if sh.isfile(filename):
cache_key = "f:%s:%s" % (sh.abspth(filename), sh.getsize(filename))
with REQUIREMENT_FILE_CACHE_LOCK:
try:
reqs = REQUIREMENT_FILE_CACHE[cache_key]
except KeyError:
reqs = tuple(pip_req.parse_requirements(filename,
session=session))
REQUIREMENT_FILE_CACHE[cache_key] = reqs
pip_requirements.extend(reqs)
return (pip_requirements,
[req.req for req in pip_requirements])
def _uninstall_dirs(self):
dirs_made = self.tracereader.dirs_made()
if dirs_made:
dirs_made = [sh.abspth(d) for d in dirs_made]
if self.get_option('keep_old', False):
download_places = [path_location[0] for path_location in self.tracereader.download_locations()]
if download_places:
utils.log_iterable(download_places, logger=LOG,
header="Keeping %s download directories (and there children directories)" % (len(download_places)))
for download_place in download_places:
dirs_made = sh.remove_parents(download_place, dirs_made)
if dirs_made:
utils.log_iterable(dirs_made, logger=LOG,
header="Removing %s created directories" % (len(dirs_made)))
for dir_name in dirs_made:
if sh.isdir(dir_name):
sh.deldir(dir_name, run_as_root=True)
else:
LOG.warn("No directory found at %s - skipping", colorizer.quote(dir_name, quote_color='red'))
def load_file(cls, fn):
persona_fn = sh.abspth(fn)
LOG.audit("Loading persona from file %r", persona_fn)
cls_kvs = None
try:
with open(persona_fn, "r") as fh:
cls_kvs = yaml.load(fh.read())
except (IOError, yaml.YAMLError) as err:
LOG.warning('Could not load persona definition from %s: %s',
persona_fn, err)
instance = None
if cls_kvs is not None:
try:
cls_kvs['source'] = persona_fn
instance = cls(**cls_kvs)
except Exception as err:
LOG.warning('Could not initialize instance %s using parameter map %s: %s',
cls, cls_kvs, err)
return instance
def apply_patches(patch_files, working_dir):
if not patch_files:
return
apply_files = []
for p in patch_files:
p = sh.abspth(p)
if not sh.isfile(p):
LOG.warn("Can not apply non-file patch %s", p)
apply_files.append(p)
if not apply_files:
return
if not sh.isdir(working_dir):
LOG.warn("Can only apply %s patches 'inside' a directory and not '%s'",
len(apply_files), working_dir)
return
with utils.chdir(working_dir):
for p in apply_files:
LOG.debug("Applying patch %s in directory %s", p, working_dir)
patch_contents = sh.load_file(p)
if patch_contents:
sh.execute(*PATCH_CMD, process_input=patch_contents)
def package_finish(self):
super(VenvDependencyHandler, self).package_finish()
for instance in self.instances:
if not self._is_buildable(instance):
continue
venv_dir = sh.abspth(self._venv_directory_for(instance))
release = str(instance.get_option("release", default_value=1))
if release and not release.startswith('-'):
release = '-' + release
version_full = instance.egg_info['version'] + release
# Replace paths with virtualenv deployment directory.
if self.opts.get('venv_deploy_dir'):
deploy_dir = sh.joinpths(self.opts.get('venv_deploy_dir'),
instance.name)
replacer = functools.partial(
re.subn, re.escape(instance.get_option('component_dir')),
deploy_dir)
bin_dir = sh.joinpths(venv_dir, 'bin')
adjustments, files_replaced = self._replace_deployment_paths(
bin_dir, replacer)
if files_replaced:
LOG.info("Adjusted %s deployment path(s) in %s files",
adjustments, files_replaced)
tar_path = sh.joinpths(
self.opts.get('venv_deploy_dir'), '%s/%s-%s-venv/venv' %
(instance.name, instance.name, version_full))
else:
tar_path = '%s/%s-%s-venv/venv' % (instance.name,
instance.name, version_full)
# Create a tarball containing the virtualenv.
tar_filename = sh.joinpths(
venv_dir, '%s-%s-venv.tar.gz' % (instance.name, version_full))
LOG.info("Making tarball of %s built for %s with version %s at %s",
venv_dir, instance.name, version_full, tar_filename)
utils.time_it(functools.partial(_on_finish, "Tarball creation"),
self._make_tarball, venv_dir, tar_filename, tar_path)
def package_finish(self):
super(VenvDependencyHandler, self).package_finish()
for instance in self.instances:
if not self._is_buildable(instance):
continue
venv_dir = sh.abspth(self._venv_directory_for(instance))
release = str(instance.get_option("release", default_value=1))
if release and not release.startswith('-'):
release = '-' + release
version_full = instance.egg_info['version'] + release
# Replace paths with virtualenv deployment directory.
if self.opts.get('venv_deploy_dir'):
deploy_dir = sh.joinpths(self.opts.get('venv_deploy_dir'),
instance.name)
replacer = functools.partial(
re.subn, re.escape(instance.get_option('component_dir')),
deploy_dir)
bin_dir = sh.joinpths(venv_dir, 'bin')
adjustments, files_replaced = self._replace_deployment_paths(bin_dir,
replacer)
if files_replaced:
LOG.info("Adjusted %s deployment path(s) in %s files",
adjustments, files_replaced)
tar_path = sh.joinpths(self.opts.get('venv_deploy_dir'), '%s/%s-%s-venv/venv' % (
instance.name, instance.name, version_full))
else:
tar_path = '%s/%s-%s-venv/venv' % (instance.name, instance.name, version_full)
# Create a tarball containing the virtualenv.
tar_filename = sh.joinpths(venv_dir, '%s-%s-venv.tar.gz' % (instance.name,
version_full))
LOG.info("Making tarball of %s built for %s with version %s at %s", venv_dir,
instance.name, version_full, tar_filename)
utils.time_it(functools.partial(_on_finish, "Tarball creation"),
self._make_tarball, venv_dir, tar_filename, tar_path)
def _begin_start(self, name, program, args):
run_trace = tr.TraceWriter(tr.trace_fn(self.runtime.get_option('trace_dir'), SCREEN_TEMPL % (name)))
run_trace.trace(NAME, name)
run_trace.trace(ARGS, json.dumps(args))
full_cmd = [program] + list(args)
session_name = self._get_session()
inited_screen = False
if session_name is None:
inited_screen = True
self._do_screen_init()
session_name = self._get_session()
if session_name is None:
msg = "After initializing screen with session named %r, no screen session with that name was found!" % (SESSION_NAME)
raise excp.StartException(msg)
run_trace.trace(SESSION_ID, session_name)
if inited_screen or not sh.isfile(SCREEN_RC):
rc_gen = ScreenRcGenerator(self)
rc_contents = rc_gen.create(session_name, self._get_env())
out_fn = sh.abspth(SCREEN_RC)
LOG.info("Writing your created screen rc file to %r" % (out_fn))
sh.write_file(out_fn, rc_contents)
self._do_start(session_name, name, full_cmd)
return run_trace.filename()
def run(args):
"""Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
if re.match(r"^moo[o]*$", action):
return
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
# Check persona file exists
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" %
(persona_fn))
# Check origin file exists
origins_fn = args.pop('origins_fn')
if not origins_fn:
raise excp.OptionException("No origin file name specified!")
if not sh.isfile(origins_fn):
raise excp.OptionException("Invalid origin file %r specified!" %
(origins_fn))
args['origins_fn'] = sh.abspth(origins_fn)
# Determine the root directory...
root_dir = sh.abspth(args.pop("dir"))
(repeat_string, line_max_len) = utils.welcome()
print(pprint.center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Ensure the anvil dirs are there if others are about to use it...
if not sh.isdir(root_dir):
LOG.info("Creating anvil root directory at path: %s", root_dir)
sh.mkdir(root_dir)
try:
for d in ANVIL_DIRS:
if sh.isdir(d):
continue
LOG.info("Creating anvil auxiliary directory at path: %s", d)
sh.mkdir(d)
except OSError as e:
LOG.warn("Failed ensuring auxiliary directories due to %s", e)
# Load the origins...
origins = _origins.load(args['origins_fn'],
patch_file=args.get('origins_patch'))
# Load the distro/s
possible_distros = distro.load(settings.DISTRO_DIR,
distros_patch=args.get('distros_patch'))
# Load + match the persona to the possible distros...
try:
persona_obj = persona.load(persona_fn)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" %
(persona_fn, e))
else:
dist = persona_obj.match(possible_distros, origins)
LOG.info('Persona selected distro: %s from %s possible distros',
colorizer.quote(dist.name), len(possible_distros))
# Update the dist with any other info...
dist.inject_platform_overrides(persona_obj.distro_updates,
source=persona_fn)
dist.inject_platform_overrides(origins, source=origins_fn)
# Print it out...
LOG.debug("Distro settings are:")
for line in dist.pformat(item_max_len=128).splitlines():
LOG.debug(line)
# Get the object we will be running with...
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("Using origins: %s", colorizer.quote(origins_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']),
colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def run(args):
"""Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
if re.match(r"^moo[o]*$", action):
return
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
# Check persona file exists
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" % (persona_fn))
# Check origin file exists
origins_fn = args.pop('origins_fn')
if not origins_fn:
raise excp.OptionException("No origin file name specified!")
if not sh.isfile(origins_fn):
raise excp.OptionException("Invalid origin file %r specified!" % (origins_fn))
args['origins_fn'] = sh.abspth(origins_fn)
# Determine the root directory...
root_dir = sh.abspth(args.pop("dir"))
(repeat_string, line_max_len) = utils.welcome()
print(pprint.center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Ensure the anvil dirs are there if others are about to use it...
if not sh.isdir(root_dir):
LOG.info("Creating anvil root directory at path: %s", root_dir)
sh.mkdir(root_dir)
try:
for d in ANVIL_DIRS:
if sh.isdir(d):
continue
LOG.info("Creating anvil auxiliary directory at path: %s", d)
sh.mkdir(d)
except OSError as e:
LOG.warn("Failed ensuring auxiliary directories due to %s", e)
# Load the origins...
origins = _origins.load(args['origins_fn'],
patch_file=args.get('origins_patch'))
# Load the distro/s
possible_distros = distro.load(settings.DISTRO_DIR,
distros_patch=args.get('distros_patch'))
# Load + match the persona to the possible distros...
try:
persona_obj = persona.load(persona_fn)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" % (persona_fn, e))
else:
dist = persona_obj.match(possible_distros, origins)
LOG.info('Persona selected distro: %s from %s possible distros',
colorizer.quote(dist.name), len(possible_distros))
# Update the dist with any other info...
dist.inject_platform_overrides(persona_obj.distro_updates, source=persona_fn)
dist.inject_platform_overrides(origins, source=origins_fn)
# Print it out...
LOG.debug("Distro settings are:")
for line in dist.pformat(item_max_len=128).splitlines():
LOG.debug(line)
# Get the object we will be running with...
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("Using origins: %s", colorizer.quote(origins_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']), colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def _build_dependencies(self):
(pips_downloaded, package_files) = self.download_dependencies()
# Analyze what was downloaded and eject things that were downloaded
# by pip as a dependency of a download but which we do not want to
# build or can satisfy by other means
no_pips = [pkg_resources.Requirement.parse(name).key
for name in self.python_names]
yum_map = self._get_known_yum_packages()
pips_keys = set([p.key for p in pips_downloaded])
def _filter_package_files(package_files):
package_reqs = []
package_keys = []
for filename in package_files:
package_details = pip_helper.get_archive_details(filename)
package_reqs.append(package_details['req'])
package_keys.append(package_details['req'].key)
package_rpm_names = self._convert_names_python2rpm(package_keys)
filtered_files = []
for (filename, req, rpm_name) in zip(package_files, package_reqs,
package_rpm_names):
if req.key in no_pips:
LOG.info(("Dependency %s was downloaded additionally "
"but it is disallowed."), colorizer.quote(req))
continue
if req.key in pips_keys:
filtered_files.append(filename)
continue
# See if pip tried to download it but we already can satisfy
# it via yum and avoid building it in the first place...
(_version, repo) = self._find_yum_match(yum_map, req, rpm_name)
if not repo:
filtered_files.append(filename)
else:
LOG.info(("Dependency %s was downloaded additionally "
"but it can be satisfied by %s from repository "
"%s instead."), colorizer.quote(req),
colorizer.quote(rpm_name),
colorizer.quote(repo))
return filtered_files
LOG.info("Filtering %s downloaded files.", len(package_files))
filtered_package_files = _filter_package_files(package_files)
if not filtered_package_files:
LOG.info("No SRPM package dependencies to build.")
return
for filename in package_files:
if filename not in filtered_package_files:
sh.unlink(filename)
package_files = sorted(filtered_package_files)
# Now build them into SRPM rpm files.
(_fn, content) = utils.load_template(sh.joinpths("packaging", "makefiles"), "source.mk")
scripts_dir = sh.abspth(sh.joinpths(settings.TEMPLATE_DIR, "packaging", "scripts"))
py2rpm_options = self._py2rpm_start_cmdline()[1:] + [
"--scripts-dir", scripts_dir,
"--source-only",
"--rpm-base", self.rpmbuild_dir,
]
params = {
"DOWNLOADS_DIR": self.download_dir,
"LOGS_DIR": self.log_dir,
"PY2RPM": self.py2rpm_executable,
"PY2RPM_FLAGS": " ".join(py2rpm_options),
}
marks_dir = sh.joinpths(self.deps_dir, "marks-deps")
if not sh.isdir(marks_dir):
sh.mkdirslist(marks_dir, tracewriter=self.tracewriter)
makefile_path = sh.joinpths(self.deps_dir, "deps.mk")
sh.write_file(makefile_path, utils.expand_template(content, params),
tracewriter=self.tracewriter)
utils.log_iterable(package_files,
header="Building %s SRPM packages using %s jobs" % (len(package_files), self._jobs),
logger=LOG)
self._execute_make(makefile_path, marks_dir)
def main():
if len(sys.argv) < 3:
print("%s distro_yaml root_dir ..." % sys.argv[0])
return 1
root_dirs = sys.argv[2:]
yaml_fn = sh.abspth(sys.argv[1])
requires_files = []
for d in root_dirs:
all_contents = sh.listdir(d, recursive=True, files_only=True)
requires_files = [
sh.abspth(f) for f in all_contents
if re.search(r"(test|pip)[-]requires$", f, re.I)
]
requires_files = sorted(list(set(requires_files)))
requirements = []
source_requirements = {}
for fn in requires_files:
source_requirements[fn] = []
for req in pip_helper.parse_requirements(sh.load_file(fn)):
requirements.append(req.key.lower().strip())
source_requirements[fn].append(req.key.lower().strip())
print("Comparing pips/pip2pkgs in %s to those found in %s" %
(yaml_fn, root_dirs))
for fn in sorted(requires_files):
print(" + " + str(fn))
requirements = set(requirements)
print("All known requirements:")
for r in sorted(requirements):
print("+ " + str(r))
distro_yaml = utils.load_yaml(yaml_fn)
components = distro_yaml.get('components', {})
all_known_names = []
components_pips = {}
for (c, details) in components.items():
components_pips[c] = []
pip2pkgs = details.get('pip_to_package', [])
pips = details.get('pips', [])
known_names = []
for item in pip2pkgs:
known_names.append(item['name'].lower().strip())
for item in pips:
known_names.append(item['name'].lower().strip())
components_pips[c].extend(known_names)
all_known_names.extend(known_names)
all_known_names = sorted(list(set(all_known_names)))
not_needed = []
for n in all_known_names:
if n not in requirements:
not_needed.append(n)
if not_needed:
print("The following distro yaml mappings may not be needed:")
for n in sorted(not_needed):
msg = " + %s (" % (n)
# Find which components said they need this...
for (c, known_names) in components_pips.items():
if n in known_names:
msg += c + ","
msg += ")"
print(msg)
not_found = []
for n in requirements:
name = n.lower().strip()
if name not in all_known_names:
not_found.append(name)
not_found = sorted(list(set(not_found)))
if not_found:
print(
"The following distro yaml mappings may be required but were not found:"
)
for n in sorted(not_found):
msg = " + %s" % (n)
msg += " ("
# Find which file/s said they need this...
for (fn, reqs) in source_requirements.items():
matched = False
for r in reqs:
if r.lower().strip() == name:
matched = True
if matched:
msg += fn + ","
msg += ")"
print(msg)
return len(not_found) + len(not_needed)
def run(args):
"""
Starts the execution after args have been parsed and logging has been setup.
Arguments: N/A
Returns: True for success to run, False for failure to start
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
if action not in actions.names():
raise excp.OptionException("Invalid action name %r specified!" %
(action))
# Determine + setup the root directory...
# If not provided attempt to locate it via the environment control files
args_root_dir = args.pop("dir")
root_dir = env.get_key('INSTALL_ROOT')
if not root_dir:
root_dir = args_root_dir
if not root_dir:
root_dir = sh.joinpths(sh.gethomedir(), 'openstack')
root_dir = sh.abspth(root_dir)
sh.mkdir(root_dir)
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" %
(persona_fn))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Stash the dryrun value (if any)
if 'dryrun' in args:
env.set("ANVIL_DRYRUN", str(args['dryrun']))
# Ensure the anvil etc dir is there if others are about to use it
ensure_anvil_dir()
# Load the distro
dist = distro.load(settings.DISTRO_DIR)
# Load + verify the person
try:
persona_obj = persona.load(persona_fn)
persona_obj.verify(dist)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" %
(persona_fn, e))
# Get the object we will be running with...
runner_cls = actions.class_for(action)
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
(repeat_string, line_max_len) = utils.welcome()
print(center_text("Action Runner", repeat_string, line_max_len))
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']),
colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def _build_dependencies(self):
(pips_downloaded, package_files) = self.download_dependencies()
# Analyze what was downloaded and eject things that were downloaded
# by pip as a dependency of a download but which we do not want to
# build or can satisfy by other means
no_pips = [
pkg_resources.Requirement.parse(name).key
for name in self.python_names
]
yum_map = self._get_known_yum_packages()
pips_keys = set([p.key for p in pips_downloaded])
def _filter_package_files(package_files):
package_reqs = []
package_keys = []
for filename in package_files:
package_details = pip_helper.get_archive_details(filename)
package_reqs.append(package_details['req'])
package_keys.append(package_details['req'].key)
package_rpm_names = self._convert_names_python2rpm(package_keys)
filtered_files = []
for (filename, req, rpm_name) in zip(package_files, package_reqs,
package_rpm_names):
if req.key in no_pips:
LOG.info(("Dependency %s was downloaded additionally "
"but it is disallowed."), colorizer.quote(req))
continue
if req.key in pips_keys:
filtered_files.append(filename)
continue
# See if pip tried to download it but we already can satisfy
# it via yum and avoid building it in the first place...
(_version, repo) = self._find_yum_match(yum_map, req, rpm_name)
if not repo:
filtered_files.append(filename)
else:
LOG.info(("Dependency %s was downloaded additionally "
"but it can be satisfied by %s from repository "
"%s instead."), colorizer.quote(req),
colorizer.quote(rpm_name), colorizer.quote(repo))
return filtered_files
LOG.info("Filtering %s downloaded files.", len(package_files))
filtered_package_files = _filter_package_files(package_files)
if not filtered_package_files:
LOG.info("No SRPM package dependencies to build.")
return
for filename in package_files:
if filename not in filtered_package_files:
sh.unlink(filename)
package_files = sorted(filtered_package_files)
# Now build them into SRPM rpm files.
(_fn,
content) = utils.load_template(sh.joinpths("packaging", "makefiles"),
"source.mk")
scripts_dir = sh.abspth(
sh.joinpths(settings.TEMPLATE_DIR, "packaging", "scripts"))
py2rpm_options = self._py2rpm_start_cmdline()[1:] + [
"--scripts-dir",
scripts_dir,
"--source-only",
"--rpm-base",
self.rpmbuild_dir,
]
params = {
"DOWNLOADS_DIR": self.download_dir,
"LOGS_DIR": self.log_dir,
"PY2RPM": self.py2rpm_executable,
"PY2RPM_FLAGS": " ".join(py2rpm_options),
}
marks_dir = sh.joinpths(self.deps_dir, "marks-deps")
if not sh.isdir(marks_dir):
sh.mkdirslist(marks_dir, tracewriter=self.tracewriter)
makefile_path = sh.joinpths(self.deps_dir, "deps.mk")
sh.write_file(makefile_path,
utils.expand_template(content, params),
tracewriter=self.tracewriter)
utils.log_iterable(package_files,
header="Building %s SRPM packages using %s jobs" %
(len(package_files), self._jobs),
logger=LOG)
self._execute_make(makefile_path, marks_dir)
def run(args):
"""Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", "").strip().lower()
if re.match(r"^moo[o]*$", action):
return
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
# Check persona file exists
persona_fn = args.pop("persona_fn")
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" % (persona_fn))
# Check origin file exists
origins_fn = args.pop("origins_fn")
if not origins_fn:
raise excp.OptionException("No origin file name specified!")
if not sh.isfile(origins_fn):
raise excp.OptionException("Invalid origin file %r specified!" % (origins_fn))
args["origins_fn"] = sh.abspth(origins_fn)
# Determine the root directory...
root_dir = sh.abspth(args.pop("dir"))
(repeat_string, line_max_len) = utils.welcome()
print(pprint.center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Stash the dryrun value (if any)
if "dryrun" in args:
sh.set_dry_run(args["dryrun"])
# Ensure the anvil dirs are there if others are about to use it...
ensure_anvil_dirs(root_dir)
# Load the distro
dist = distro.load(settings.DISTRO_DIR)
# Load + verify the person
try:
persona_obj = persona.load(persona_fn)
persona_obj.verify(dist)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" % (persona_fn, e))
yum.YumDependencyHandler.jobs = args["jobs"]
# Get the object we will be running with...
runner = runner_cls(distro=dist, root_dir=root_dir, name=action, cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info(
"Starting action %s on %s for distro: %s",
colorizer.quote(action),
colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name),
)
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("Using origins: %s", colorizer.quote(origins_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info(
"It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time["seconds"]),
colorizer.quote(pretty_time["minutes"]),
colorizer.quote(action),
)
def get_directory_details(path, pbr_version=None):
if not sh.isdir(path):
raise IOError("Can not detail non-existent directory %s" % (path))
# Check if we already got the details of this dir previously
with EGGS_DETAILED_LOCK:
path = sh.abspth(path)
cache_key = "d:%s" % (sh.abspth(path))
if cache_key in EGGS_DETAILED:
return EGGS_DETAILED[cache_key]
details = None
skip_paths = [
sh.joinpths(path, "PKG-INFO"),
sh.joinpths(path, "EGG-INFO"),
]
skip_paths.extend(glob.glob(sh.joinpths(path, "*.egg-info")))
if any(sh.exists(a_path) for a_path in skip_paths):
# Some packages seem to not support the 'egg_info' call and
# provide there own path/file that contains this information
# already, so just use it if we can get at it...
#
# Ie for pyyaml3.x:
#
# error: invalid command 'egg_info'
details = pkginfo.Develop(path)
if not details or not details.name:
cmd = [sys.executable, 'setup.py', 'egg_info']
if pbr_version:
env_overrides = {
"PBR_VERSION": str(pbr_version),
}
else:
env_overrides = {}
sh.execute(cmd, cwd=path, env_overrides=env_overrides)
details = pkginfo.get_metadata(path)
if not details or not details.name:
raise RuntimeError("No egg detail information discovered"
" at '%s'" % path)
egg_details = {
'req': create_requirement(details.name, version=details.version),
}
for attr_name in ['description', 'author',
'version', 'name', 'summary']:
egg_details[attr_name] = getattr(details, attr_name)
for attr_name in ['description', 'author', 'summary']:
attr_value = egg_details[attr_name]
if isinstance(attr_value, six.text_type):
# Fix any unicode which will cause unicode decode failures...
# versions or names shouldn't be unicode, and the rest
# we don't really care about being unicode (since its
# just used for logging right now anyway...).
#
# The reason this is done is that 'elasticsearch' seems to
# have a unicode author name, and that causes the log_object
# to blowup, so just avoid that by replacing this information
# in the first place.
egg_details[attr_name] = attr_value.encode("ascii",
errors='replace')
LOG.debug("Extracted '%s' egg detail information:", path)
utils.log_object(egg_details, logger=LOG, level=logging.DEBUG)
EGGS_DETAILED[cache_key] = egg_details
return egg_details
def main():
if len(sys.argv) < 3:
print("%s distro_yaml root_dir ..." % sys.argv[0])
return 1
root_dirs = sys.argv[2:]
yaml_fn = sh.abspth(sys.argv[1])
requires_files = []
for d in root_dirs:
all_contents = sh.listdir(d, recursive=True, files_only=True)
requires_files = [sh.abspth(f) for f in all_contents
if re.search(r"(test|pip)[-]requires$", f, re.I)]
requires_files = sorted(list(set(requires_files)))
requirements = []
source_requirements = {}
for fn in requires_files:
source_requirements[fn] = []
for req in pip_helper.parse_requirements(sh.load_file(fn)):
requirements.append(req.key.lower().strip())
source_requirements[fn].append(req.key.lower().strip())
print("Comparing pips/pip2pkgs in %s to those found in %s" % (yaml_fn, root_dirs))
for fn in sorted(requires_files):
print(" + " + str(fn))
requirements = set(requirements)
print("All known requirements:")
for r in sorted(requirements):
print("+ " + str(r))
distro_yaml = utils.load_yaml(yaml_fn)
components = distro_yaml.get('components', {})
all_known_names = []
components_pips = {}
for (c, details) in components.items():
components_pips[c] = []
pip2pkgs = details.get('pip_to_package', [])
pips = details.get('pips', [])
known_names = []
for item in pip2pkgs:
known_names.append(item['name'].lower().strip())
for item in pips:
known_names.append(item['name'].lower().strip())
components_pips[c].extend(known_names)
all_known_names.extend(known_names)
all_known_names = sorted(list(set(all_known_names)))
not_needed = []
for n in all_known_names:
if n not in requirements:
not_needed.append(n)
if not_needed:
print("The following distro yaml mappings may not be needed:")
for n in sorted(not_needed):
msg = " + %s (" % (n)
# Find which components said they need this...
for (c, known_names) in components_pips.items():
if n in known_names:
msg += c + ","
msg += ")"
print(msg)
not_found = []
for n in requirements:
name = n.lower().strip()
if name not in all_known_names:
not_found.append(name)
not_found = sorted(list(set(not_found)))
if not_found:
print("The following distro yaml mappings may be required but were not found:")
for n in sorted(not_found):
msg = " + %s" % (n)
msg += " ("
# Find which file/s said they need this...
for (fn, reqs) in source_requirements.items():
matched = False
for r in reqs:
if r.lower().strip() == name:
matched = True
if matched:
msg += fn + ","
msg += ")"
print(msg)
return len(not_found) + len(not_needed)
def _build_dependencies(self):
(pips_downloaded, package_files) = self.download_dependencies()
# Analyze what was downloaded and eject things that were downloaded
# by pip as a dependency of a download but which we do not want to
# build or can satisfy by other means
no_pips = [pkg_resources.Requirement.parse(name).key
for name in self.python_names]
no_pips.extend(self.BANNED_PACKAGES)
yum_map = self._get_yum_available()
pips_keys = set([p.key for p in pips_downloaded])
def _filter_package_files(package_files):
package_reqs = []
package_keys = []
for filename in package_files:
package_details = pip_helper.get_archive_details(filename)
package_reqs.append(package_details['req'])
package_keys.append(package_details['req'].key)
package_rpm_names = self._convert_names_python2rpm(package_keys)
filtered_files = []
for (filename, req, rpm_name) in zip(package_files, package_reqs,
package_rpm_names):
if req.key in no_pips:
LOG.info(("Dependency %s was downloaded additionally "
"but it is disallowed."), colorizer.quote(req))
continue
if req.key in pips_keys:
filtered_files.append(filename)
continue
# See if pip tried to download it but we already can satisfy
# it via yum and avoid building it in the first place...
(_version, repo) = self._find_yum_match(yum_map, req, rpm_name)
if not repo:
filtered_files.append(filename)
else:
LOG.info(("Dependency %s was downloaded additionally "
"but it can be satisfied by %s from repository "
"%s instead."), colorizer.quote(req),
colorizer.quote(rpm_name),
colorizer.quote(repo))
return filtered_files
LOG.info("Filtering %s downloaded files.", len(package_files))
package_files = _filter_package_files(package_files)
if not package_files:
LOG.info("No SRPM package dependencies to build.")
return
deps_makefile_name = sh.joinpths(self.deps_dir, "deps.mk")
marks_dir = sh.joinpths(self.deps_dir, "marks-deps")
sh.mkdirslist(marks_dir)
with open(deps_makefile_name, "w") as makefile:
scripts_dir = sh.abspth(sh.joinpths(settings.TEMPLATE_DIR, "packaging", "scripts"))
py2rpm_options = self.py2rpm_options() + [
"--scripts-dir", scripts_dir,
"--source-only",
"--rpm-base", self.rpmbuild_dir,
]
print >> makefile, "DOWNLOADS_DIR :=", self.download_dir
print >> makefile, "LOGS_DIR :=", self.log_dir
print >> makefile, "PY2RPM :=", self.py2rpm_executable
print >> makefile, "PY2RPM_FLAGS :=", " ".join(py2rpm_options)
print >> makefile, """
%.mark: $(DOWNLOADS_DIR)/%
\t@$(PY2RPM) $(PY2RPM_FLAGS) -- $^ &> $(LOGS_DIR)/py2rpm-$*.log
\t@touch "$@"
\t@echo "$* is processed"
"""
print >> makefile, "MARKS :=", " ".join(
"%s.mark" % sh.basename(i) for i in package_files)
print >> makefile
print >> makefile, "all: $(MARKS)"
base_package_files = [sh.basename(f) for f in package_files]
utils.log_iterable(base_package_files,
header="Building %s SRPM packages using %s jobs" %
(len(package_files), self.jobs),
logger=LOG)
self._execute_make(deps_makefile_name, marks_dir)
def run(args):
"""Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
if re.match(r"^moo[o]*$", action):
return
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" %
(persona_fn))
# Determine the root directory...
root_dir = sh.abspth(args.pop("dir"))
(repeat_string, line_max_len) = utils.welcome()
print(pprint.center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Stash the dryrun value (if any)
if 'dryrun' in args:
sh.set_dry_run(args['dryrun'])
# Ensure the anvil dirs are there if others are about to use it...
ensure_anvil_dirs(root_dir)
# Load the distro
dist = distro.load(settings.DISTRO_DIR)
# Load + verify the person
try:
persona_obj = persona.load(persona_fn)
persona_obj.verify(dist)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" %
(persona_fn, e))
yum.YumDependencyHandler.jobs = args["jobs"]
# Get the object we will be running with...
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']),
colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def run(args):
"""
Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" % (persona_fn))
# Determine + setup the root directory...
# If not provided attempt to locate it via the environment control files
args_root_dir = args.pop("dir")
root_dir = env.get_key('INSTALL_ROOT')
if not root_dir:
root_dir = args_root_dir
if not root_dir:
root_dir = sh.joinpths(sh.gethomedir(), 'openstack')
root_dir = sh.abspth(root_dir)
(repeat_string, line_max_len) = utils.welcome()
print(center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Stash the dryrun value (if any)
if 'dryrun' in args:
sh.set_dry_run(args['dryrun'])
# Ensure the anvil dirs are there if others are about to use it...
ensure_anvil_dirs(root_dir)
# Load the distro
dist = distro.load(settings.DISTRO_DIR)
# Load + verify the person
try:
persona_obj = persona.load(persona_fn)
persona_obj.verify(dist)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" % (persona_fn, e))
# Get the object we will be running with...
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']), colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def get_directory_details(path, pbr_version=None):
if not sh.isdir(path):
raise IOError("Can not detail non-existent directory %s" % (path))
# Check if we already got the details of this dir previously
with EGGS_DETAILED_LOCK:
path = sh.abspth(path)
cache_key = "d:%s" % (sh.abspth(path))
if cache_key in EGGS_DETAILED:
return EGGS_DETAILED[cache_key]
details = None
skip_paths = [
sh.joinpths(path, "PKG-INFO"),
sh.joinpths(path, "EGG-INFO"),
]
skip_paths.extend(glob.glob(sh.joinpths(path, "*.egg-info")))
if any(sh.exists(a_path) for a_path in skip_paths):
# Some packages seem to not support the 'egg_info' call and
# provide there own path/file that contains this information
# already, so just use it if we can get at it...
#
# Ie for pyyaml3.x:
#
# error: invalid command 'egg_info'
details = pkginfo.Develop(path)
if not details or not details.name:
cmd = [sys.executable, 'setup.py', 'egg_info']
if pbr_version:
env_overrides = {
"PBR_VERSION": str(pbr_version),
}
else:
env_overrides = {}
sh.execute(cmd, cwd=path, env_overrides=env_overrides)
details = pkginfo.get_metadata(path)
if not details or not details.name:
raise RuntimeError("No egg detail information discovered"
" at '%s'" % path)
egg_details = {
'req': create_requirement(details.name, version=details.version),
}
for attr_name in [
'description', 'author', 'version', 'name', 'summary'
]:
egg_details[attr_name] = getattr(details, attr_name)
for attr_name in ['description', 'author', 'summary']:
attr_value = egg_details[attr_name]
if isinstance(attr_value, six.text_type):
# Fix any unicode which will cause unicode decode failures...
# versions or names shouldn't be unicode, and the rest
# we don't really care about being unicode (since its
# just used for logging right now anyway...).
#
# The reason this is done is that 'elasticsearch' seems to
# have a unicode author name, and that causes the log_object
# to blowup, so just avoid that by replacing this information
# in the first place.
egg_details[attr_name] = attr_value.encode("ascii",
errors='replace')
LOG.debug("Extracted '%s' egg detail information:", path)
utils.log_object(egg_details, logger=LOG, level=logging.DEBUG)
EGGS_DETAILED[cache_key] = egg_details
return egg_details
