def _create_package(self):
files = self._gather_files()
params = {
'files': files,
'requires': self._requirements(),
'obsoletes': self._obsoletes(),
'conflicts': self._conflicts(),
'defines': self._defines(),
'undefines': self._undefines(),
'build': self._build_details(),
'who': sh.getuser(),
'date': utils.iso8601(),
'details': self.details,
}
(_fn, content) = utils.load_template('packaging', 'spec.tmpl')
spec_base = self._make_fn("spec")
spec_fn = sh.joinpths(self.build_paths['specs'], spec_base)
LOG.debug("Creating spec file %s with params:", spec_fn)
files['sources'].append("%s.tar.gz" % (spec_base))
utils.log_object(params, logger=LOG, level=logging.DEBUG)
sh.write_file(spec_fn, utils.expand_template(content, params))
tar_it(sh.joinpths(self.build_paths['sources'],
"%s.tar.gz" % (spec_base)),
spec_base,
wkdir=self.build_paths['specs'])
def _create_package(self):
files = self._gather_files()
params = {
"files": files,
"requires": self._requirements(),
"obsoletes": self._obsoletes(),
"conflicts": self._conflicts(),
"defines": self._defines(),
"undefines": self._undefines(),
"build": self._build_details(),
"who": sh.getuser(),
"date": utils.iso8601(),
"patches": self._patches(),
"details": self.details,
}
(_fn, content) = utils.load_template("packaging", "spec.tmpl")
spec_base = self._make_fn("spec")
spec_fn = sh.joinpths(self.build_paths["specs"], spec_base)
LOG.debug("Creating spec file %s with params:", spec_fn)
files["sources"].append("%s.tar.gz" % (spec_base))
utils.log_object(params, logger=LOG, level=logging.DEBUG)
sh.write_file(spec_fn, utils.expand_template(content, params))
tar_it(
sh.joinpths(self.build_paths["sources"], "%s.tar.gz" % (spec_base)),
spec_base,
wkdir=self.build_paths["specs"],
)
def _clean_pip_requires(self):
# Fixup these files if they exist (sometimes they have 'junk' in them)
req_fns = []
for fn in self.requires_files:
if not sh.isfile(fn):
continue
req_fns.append(fn)
if req_fns:
utils.log_iterable(req_fns, logger=LOG,
header="Adjusting %s pip 'requires' files" % (len(req_fns)))
for fn in req_fns:
new_lines = []
for line in sh.load_file(fn).splitlines():
s_line = line.strip()
if len(s_line) == 0:
continue
elif s_line.startswith("#"):
new_lines.append(s_line)
elif not self._filter_pip_requires_line(s_line):
new_lines.append(("# %s" % (s_line)))
else:
new_lines.append(s_line)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
return len(req_fns)
def replace_forced_requirements(fn, forced_by_key):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
for line in old_lines:
try:
source_req = pip_helper.extract_requirement(line)
except (ValueError, TypeError):
pass
else:
if source_req:
validate_requirement(fn, source_req)
try:
replace_req = forced_by_key[source_req.key]
except KeyError:
pass
else:
replace_req = str(replace_req)
source_req = str(source_req)
if replace_req != source_req:
line = replace_req
alterations.append("%s => %s"
% (colorizer.quote(source_req),
colorizer.quote(replace_req)))
new_lines.append(line)
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s"
% (len(alterations), fn),
color=None)
return len(alterations)
def _clean_pip_requires(self):
# Fixup these files if they exist (sometimes they have 'junk' in them)
req_fns = []
for fn in self.requires_files:
if not sh.isfile(fn):
continue
req_fns.append(fn)
if req_fns:
utils.log_iterable(req_fns, logger=LOG,
header="Adjusting %s pip 'requires' files" % (len(req_fns)))
for fn in req_fns:
new_lines = []
for line in sh.load_file(fn).splitlines():
s_line = line.strip()
if len(s_line) == 0:
continue
elif s_line.startswith("#"):
new_lines.append(s_line)
elif not self._filter_pip_requires_line(fn, s_line):
new_lines.append(("# %s" % (s_line)))
else:
new_lines.append(s_line)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
return len(req_fns)
def mark(self, phasename):
contents = dict()
contents['name'] = phasename
contents['when'] = utils.iso8601()
yield phasename
LOG.debug("Marking the completion of phase %r in file %r", phasename, self.fn)
lines = [json.dumps(contents), '']
sh.append_file(self.fn, utils.joinlinesep(*lines))
def _clean_pip_requires(self):
# Fixup these files if they exist, sometimes they have 'junk' in them
# that anvil will install instead of pip or setup.py and we don't want
# the setup.py file to attempt to install said dependencies since it
# typically picks locations that either are not what we desire or if
# said file contains editables, it may even pick external source directories
# which is what anvil is setting up as well...
req_fns = [f for f in self.requires_files if sh.isfile(f)]
if req_fns:
utils.log_iterable(req_fns, logger=LOG, header="Adjusting %s pip 'requires' files" % (len(req_fns)))
for fn in req_fns:
old_lines = sh.load_file(fn).splitlines()
new_lines = self._filter_pip_requires(fn, old_lines)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
return len(req_fns)
def details(self):
if self._details is not None:
return self._details
self._details = {
'name': self.name,
'version': 0,
'release': self.get_int_option('release', default_value=1),
'packager': "%s <%s@%s>" % (sh.getuser(), sh.getuser(), sh.hostname()),
'changelog': '',
'license': 'Apache License, Version 2.0',
'automatic_dependencies': True,
'vendor': None,
'url': '',
'description': '',
'summary': 'Package build of %s on %s' % (self.name, utils.iso8601()),
}
return self._details
def details(self):
if self._details is not None:
return self._details
self._details = {
'name': self.name,
'version': 0,
'release': self.get_int_option('release', default_value=1),
'packager':
"%s <%s@%s>" % (sh.getuser(), sh.getuser(), sh.hostname()),
'changelog': '',
'license': 'Apache License, Version 2.0',
'automatic_dependencies': True,
'vendor': None,
'url': '',
'description': '',
'summary':
'Package build of %s on %s' % (self.name, utils.iso8601()),
}
return self._details
def _clean_pip_requires(self):
# Fixup these files if they exist, sometimes they have 'junk' in them
# that anvil will install instead of pip or setup.py and we don't want
# the setup.py file to attempt to install said dependencies since it
# typically picks locations that either are not what we desire or if
# said file contains editables, it may even pick external source directories
# which is what anvil is setting up as well...
req_fns = [f for f in self.requires_files if sh.isfile(f)]
if req_fns:
utils.log_iterable(req_fns,
logger=LOG,
header="Adjusting %s pip 'requires' files" %
(len(req_fns)))
for fn in req_fns:
old_lines = sh.load_file(fn).splitlines()
new_lines = self._filter_pip_requires(fn, old_lines)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(),
"\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
return len(req_fns)
def _clean_pip_requires(self, requires_files):
# Fixup incompatible dependencies
if not (requires_files and self.forced_packages):
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Adjusting %s pip 'requires' files" % (len(requires_files)))
forced_by_key = dict((pkg.key, pkg) for pkg in self.forced_packages)
for fn in requires_files:
old_lines = sh.load_file(fn).splitlines()
new_lines = []
for line in old_lines:
try:
req = pip_helper.extract_requirement(line)
new_lines.append(str(forced_by_key[req.key]))
except Exception:
# we don't force the package or it has a bad format
new_lines.append(line)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
def _clean_pip_requires(self, requires_files):
# Fixup incompatible dependencies
if not (requires_files and self.forced_packages):
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Adjusting %s pip 'requires' files" %
(len(requires_files)))
forced_by_key = dict((pkg.key, pkg) for pkg in self.forced_packages)
for fn in requires_files:
old_lines = sh.load_file(fn).splitlines()
new_lines = []
for line in old_lines:
try:
req = pip_helper.extract_requirement(line)
new_lines.append(str(forced_by_key[req.key]))
except Exception:
# we don't force the package or it has a bad format
new_lines.append(line)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(),
"\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
def _create_package(self):
files = self._gather_files()
params = {
'files': files,
'requires': self._requirements(),
'obsoletes': self._obsoletes(),
'conflicts': self._conflicts(),
'defines': self._defines(),
'undefines': self._undefines(),
'build': self._build_details(),
'who': sh.getuser(),
'date': utils.iso8601(),
'details': self.details,
}
(_fn, content) = utils.load_template('packaging', 'spec.tmpl')
spec_base = self._make_fn("spec")
spec_fn = sh.joinpths(self.build_paths['specs'], spec_base)
LOG.debug("Creating spec file %s with params:", spec_fn)
files['sources'].append("%s.tar.gz" % (spec_base))
utils.log_object(params, logger=LOG, level=logging.DEBUG)
sh.write_file(spec_fn, utils.expand_template(content, params))
tar_it(sh.joinpths(self.build_paths['sources'], "%s.tar.gz" % (spec_base)),
spec_base, wkdir=self.build_paths['specs'])
def replace_forced_requirements(fn, forced_by_key):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
for line in old_lines:
try:
source_req = pip_helper.extract_requirement(line)
except (ValueError, TypeError):
pass
else:
if source_req:
validate_requirement(fn, source_req)
try:
replace_req = forced_by_key[source_req.key]
except KeyError:
pass
else:
replace_req = str(replace_req)
source_req = str(source_req)
if replace_req != source_req:
line = replace_req
alterations.append(
"%s => %s" %
(colorizer.quote(source_req),
colorizer.quote(replace_req)))
new_lines.append(line)
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(),
"\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s" %
(len(alterations), fn),
color=None)
return len(alterations)
def run(args):
"""
Starts the execution after args have been parsed and logging has been setup.
Arguments: N/A
Returns: True for success to run, False for failure to start
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
if action not in actions.names():
raise excp.OptionException("Invalid action name %r specified!" %
(action))
# Determine + setup the root directory...
# If not provided attempt to locate it via the environment control files
args_root_dir = args.pop("dir")
root_dir = env.get_key('INSTALL_ROOT')
if not root_dir:
root_dir = args_root_dir
if not root_dir:
root_dir = sh.joinpths(sh.gethomedir(), 'openstack')
root_dir = sh.abspth(root_dir)
sh.mkdir(root_dir)
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" %
(persona_fn))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Stash the dryrun value (if any)
if 'dryrun' in args:
env.set("ANVIL_DRYRUN", str(args['dryrun']))
# Ensure the anvil etc dir is there if others are about to use it
ensure_anvil_dir()
# Load the distro
dist = distro.load(settings.DISTRO_DIR)
# Load + verify the person
try:
persona_obj = persona.load(persona_fn)
persona_obj.verify(dist)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" %
(persona_fn, e))
# Get the object we will be running with...
runner_cls = actions.class_for(action)
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
(repeat_string, line_max_len) = utils.welcome()
print(center_text("Action Runner", repeat_string, line_max_len))
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']),
colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def run(args):
"""Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", "").strip().lower()
if re.match(r"^moo[o]*$", action):
return
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
# Check persona file exists
persona_fn = args.pop("persona_fn")
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" % (persona_fn))
# Check origin file exists
origins_fn = args.pop("origins_fn")
if not origins_fn:
raise excp.OptionException("No origin file name specified!")
if not sh.isfile(origins_fn):
raise excp.OptionException("Invalid origin file %r specified!" % (origins_fn))
args["origins_fn"] = sh.abspth(origins_fn)
# Determine the root directory...
root_dir = sh.abspth(args.pop("dir"))
(repeat_string, line_max_len) = utils.welcome()
print(pprint.center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Stash the dryrun value (if any)
if "dryrun" in args:
sh.set_dry_run(args["dryrun"])
# Ensure the anvil dirs are there if others are about to use it...
ensure_anvil_dirs(root_dir)
# Load the distro
dist = distro.load(settings.DISTRO_DIR)
# Load + verify the person
try:
persona_obj = persona.load(persona_fn)
persona_obj.verify(dist)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" % (persona_fn, e))
yum.YumDependencyHandler.jobs = args["jobs"]
# Get the object we will be running with...
runner = runner_cls(distro=dist, root_dir=root_dir, name=action, cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info(
"Starting action %s on %s for distro: %s",
colorizer.quote(action),
colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name),
)
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("Using origins: %s", colorizer.quote(origins_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info(
"It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time["seconds"]),
colorizer.quote(pretty_time["minutes"]),
colorizer.quote(action),
)
def _scan_pip_requires(self, requires_files):
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in self._python_eggs:
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(msg % (source_req,
install_egg['req'],
filename))
if not requires_files:
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Scanning %s pip 'requires' files" % (len(requires_files)))
forced_by_key = dict((pkg.key, pkg) for pkg in self.forced_packages)
mutations = 0
for fn in sorted(requires_files):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
for line in old_lines:
try:
source_req = pip_helper.extract_requirement(line)
except (ValueError, TypeError):
pass
else:
if source_req:
validate_requirement(fn, source_req)
try:
replace_req = forced_by_key[source_req.key]
except KeyError:
pass
else:
replace_req = str(replace_req)
source_req = str(source_req)
if replace_req != source_req:
line = replace_req
alterations.append("%s => %s"
% (colorizer.quote(source_req),
colorizer.quote(replace_req)))
new_lines.append(line)
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
mutations += len(alterations)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s"
% (len(alterations), fn),
color=None)
# NOTE(imelnikov): after updating requirement lists we should re-fetch
# data from them again, so we drop pip helper caches here.
if mutations > 0:
pip_helper.drop_caches()
def run(args):
"""Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
if re.match(r"^moo[o]*$", action):
return
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
# Check persona file exists
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" % (persona_fn))
# Check origin file exists
origins_fn = args.pop('origins_fn')
if not origins_fn:
raise excp.OptionException("No origin file name specified!")
if not sh.isfile(origins_fn):
raise excp.OptionException("Invalid origin file %r specified!" % (origins_fn))
args['origins_fn'] = sh.abspth(origins_fn)
# Determine the root directory...
root_dir = sh.abspth(args.pop("dir"))
(repeat_string, line_max_len) = utils.welcome()
print(pprint.center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Ensure the anvil dirs are there if others are about to use it...
if not sh.isdir(root_dir):
LOG.info("Creating anvil root directory at path: %s", root_dir)
sh.mkdir(root_dir)
try:
for d in ANVIL_DIRS:
if sh.isdir(d):
continue
LOG.info("Creating anvil auxiliary directory at path: %s", d)
sh.mkdir(d)
except OSError as e:
LOG.warn("Failed ensuring auxiliary directories due to %s", e)
# Load the origins...
origins = _origins.load(args['origins_fn'],
patch_file=args.get('origins_patch'))
# Load the distro/s
possible_distros = distro.load(settings.DISTRO_DIR,
distros_patch=args.get('distros_patch'))
# Load + match the persona to the possible distros...
try:
persona_obj = persona.load(persona_fn)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" % (persona_fn, e))
else:
dist = persona_obj.match(possible_distros, origins)
LOG.info('Persona selected distro: %s from %s possible distros',
colorizer.quote(dist.name), len(possible_distros))
# Update the dist with any other info...
dist.inject_platform_overrides(persona_obj.distro_updates, source=persona_fn)
dist.inject_platform_overrides(origins, source=origins_fn)
# Print it out...
LOG.debug("Distro settings are:")
for line in dist.pformat(item_max_len=128).splitlines():
LOG.debug(line)
# Get the object we will be running with...
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("Using origins: %s", colorizer.quote(origins_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']), colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def run(args):
"""Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
if re.match(r"^moo[o]*$", action):
return
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" %
(persona_fn))
# Determine the root directory...
root_dir = sh.abspth(args.pop("dir"))
(repeat_string, line_max_len) = utils.welcome()
print(pprint.center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Stash the dryrun value (if any)
if 'dryrun' in args:
sh.set_dry_run(args['dryrun'])
# Ensure the anvil dirs are there if others are about to use it...
ensure_anvil_dirs(root_dir)
# Load the distro
dist = distro.load(settings.DISTRO_DIR)
# Load + verify the person
try:
persona_obj = persona.load(persona_fn)
persona_obj.verify(dist)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" %
(persona_fn, e))
yum.YumDependencyHandler.jobs = args["jobs"]
# Get the object we will be running with...
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']),
colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def _get_summary(self):
return 'Package build of %s on %s' % (self.name, utils.iso8601())
def _scan_pip_requires(self, requires_files):
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in self._python_eggs:
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(
msg % (source_req, install_egg['req'], filename))
if not requires_files:
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Scanning %s pip 'requires' files" %
(len(requires_files)))
forced_by_key = dict((pkg.key, pkg) for pkg in self.forced_packages)
mutations = 0
for fn in sorted(requires_files):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
for line in old_lines:
try:
source_req = pip_helper.extract_requirement(line)
except (ValueError, TypeError):
pass
else:
if source_req:
validate_requirement(fn, source_req)
try:
replace_req = forced_by_key[source_req.key]
except KeyError:
pass
else:
replace_req = str(replace_req)
source_req = str(source_req)
if replace_req != source_req:
line = replace_req
alterations.append(
"%s => %s" %
(colorizer.quote(source_req),
colorizer.quote(replace_req)))
new_lines.append(line)
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(),
"\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
mutations += len(alterations)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s" %
(len(alterations), fn),
color=None)
# NOTE(imelnikov): after updating requirement lists we should re-fetch
# data from them again, so we drop pip helper caches here.
if mutations > 0:
pip_helper.drop_caches()
def _get_summary(self):
return 'Package build of %s on %s' % (self.name, utils.iso8601())
def mark(self, what):
contents = self.list_phases()
contents[what] = utils.iso8601()
yield what
sh.write_file(self.fn, self._format_contents(contents))
def run(args):
"""
Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" % (persona_fn))
# Determine + setup the root directory...
# If not provided attempt to locate it via the environment control files
args_root_dir = args.pop("dir")
root_dir = env.get_key('INSTALL_ROOT')
if not root_dir:
root_dir = args_root_dir
if not root_dir:
root_dir = sh.joinpths(sh.gethomedir(), 'openstack')
root_dir = sh.abspth(root_dir)
(repeat_string, line_max_len) = utils.welcome()
print(center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Stash the dryrun value (if any)
if 'dryrun' in args:
sh.set_dry_run(args['dryrun'])
# Ensure the anvil dirs are there if others are about to use it...
ensure_anvil_dirs(root_dir)
# Load the distro
dist = distro.load(settings.DISTRO_DIR)
# Load + verify the person
try:
persona_obj = persona.load(persona_fn)
persona_obj.verify(dist)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" % (persona_fn, e))
# Get the object we will be running with...
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']), colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def run(args):
"""Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
if re.match(r"^moo[o]*$", action):
return
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
# Check persona file exists
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" %
(persona_fn))
# Check origin file exists
origins_fn = args.pop('origins_fn')
if not origins_fn:
raise excp.OptionException("No origin file name specified!")
if not sh.isfile(origins_fn):
raise excp.OptionException("Invalid origin file %r specified!" %
(origins_fn))
args['origins_fn'] = sh.abspth(origins_fn)
# Determine the root directory...
root_dir = sh.abspth(args.pop("dir"))
(repeat_string, line_max_len) = utils.welcome()
print(pprint.center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Ensure the anvil dirs are there if others are about to use it...
if not sh.isdir(root_dir):
LOG.info("Creating anvil root directory at path: %s", root_dir)
sh.mkdir(root_dir)
try:
for d in ANVIL_DIRS:
if sh.isdir(d):
continue
LOG.info("Creating anvil auxiliary directory at path: %s", d)
sh.mkdir(d)
except OSError as e:
LOG.warn("Failed ensuring auxiliary directories due to %s", e)
# Load the origins...
origins = _origins.load(args['origins_fn'],
patch_file=args.get('origins_patch'))
# Load the distro/s
possible_distros = distro.load(settings.DISTRO_DIR,
distros_patch=args.get('distros_patch'))
# Load + match the persona to the possible distros...
try:
persona_obj = persona.load(persona_fn)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" %
(persona_fn, e))
else:
dist = persona_obj.match(possible_distros, origins)
LOG.info('Persona selected distro: %s from %s possible distros',
colorizer.quote(dist.name), len(possible_distros))
# Update the dist with any other info...
dist.inject_platform_overrides(persona_obj.distro_updates,
source=persona_fn)
dist.inject_platform_overrides(origins, source=origins_fn)
# Print it out...
LOG.debug("Distro settings are:")
for line in dist.pformat(item_max_len=128).splitlines():
LOG.debug(line)
# Get the object we will be running with...
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("Using origins: %s", colorizer.quote(origins_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']),
colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def mark(self, what):
contents = self.list_phases()
contents[what] = utils.iso8601()
yield what
sh.write_file(self.filename, self._format_contents(contents))
