def get_suids():
uid = env.get_key('SUDO_UID')
if uid is not None:
uid = int(uid)
gid = env.get_key('SUDO_GID')
if gid is not None:
gid = int(gid)
return (uid, gid)
def get_suids():
uid = env.get_key('SUDO_UID')
if uid is not None:
uid = int(uid)
gid = env.get_key('SUDO_GID')
if gid is not None:
gid = int(gid)
return (uid, gid)
def _install_into_venv(self,
instance,
requirements,
upgrade=False,
extra_env_overrides=None):
venv_dir = self._venv_directory_for(instance)
base_pip = [sh.joinpths(venv_dir, 'bin', 'pip')]
env_overrides = {
'PATH':
os.pathsep.join([
sh.joinpths(venv_dir, "bin"),
env.get_key('PATH', default_value='')
]),
'VIRTUAL_ENV':
venv_dir,
}
if extra_env_overrides:
env_overrides.update(extra_env_overrides)
cmd = list(base_pip) + ['install']
if upgrade:
cmd.append("--upgrade")
if isinstance(requirements, six.string_types):
cmd.extend(['--requirement', requirements])
else:
for req in requirements:
cmd.append(str(req))
count = self.install_counters.get(instance.name, 0)
self.install_counters[instance.name] = count + 1
out_filename = sh.joinpths(
self.log_dir, "venv-install-%s-%s.log" % (instance.name, count))
sh.execute_save_output(cmd, out_filename, env_overrides=env_overrides)
def _install_into_venv(self, instance, requirements,
upgrade=False, extra_env_overrides=None):
venv_dir = self._venv_directory_for(instance)
base_pip = [sh.joinpths(venv_dir, 'bin', 'pip')]
env_overrides = {
'PATH': os.pathsep.join([sh.joinpths(venv_dir, "bin"),
env.get_key('PATH', default_value='')]),
'VIRTUAL_ENV': venv_dir,
}
if extra_env_overrides:
env_overrides.update(extra_env_overrides)
cmd = list(base_pip) + ['install']
if upgrade:
cmd.append("--upgrade")
if isinstance(requirements, six.string_types):
cmd.extend([
'--requirement',
requirements
])
else:
for req in requirements:
cmd.append(str(req))
count = self.install_counters.get(instance.name, 0)
self.install_counters[instance.name] = count + 1
out_filename = sh.joinpths(self.log_dir, "venv-install-%s-%s.log" % (instance.name, count))
sh.execute_save_output(cmd, out_filename, env_overrides=env_overrides)
def _install_into_venv(self, instance, requirements):
venv_dir = self._venv_directory_for(instance)
base_pip = [sh.joinpths(venv_dir, 'bin', 'pip')]
env_overrides = {
'PATH': os.pathsep.join([sh.joinpths(venv_dir, "bin"),
env.get_key('PATH', default_value='')]),
'VIRTUAL_ENV': venv_dir,
}
sh.mkdirslist(self.cache_dir, tracewriter=self.tracewriter)
def try_install(attempt, requirements):
cmd = list(base_pip) + ['install']
cmd.extend([
'--download-cache',
self.cache_dir,
])
if isinstance(requirements, six.string_types):
cmd.extend([
'--requirement',
requirements
])
else:
for req in requirements:
cmd.append(str(req))
sh.execute(cmd, env_overrides=env_overrides)
# Sometimes pip fails downloading things, retry it when this happens...
utils.retry(3, 5, try_install, requirements=requirements)
def _get_bashed(self, section, option):
value = self.backing.get(section, option)
if value is None:
return value
extracted_val = ""
mtch = ENV_PAT.match(value)
if mtch:
env_key = mtch.group(1).strip()
def_val = mtch.group(2).strip()
if not def_val and not env_key:
msg = "Invalid bash-like value %r" % (value)
raise excp.BadParamException(msg)
LOG.debug("Looking for that value in environment variable: %r", env_key)
env_value = env.get_key(env_key)
if env_value is None:
LOG.debug("Extracting value from config provided default value %r" % (def_val))
extracted_val = self._resolve_replacements(def_val)
LOG.debug("Using config provided default value %r (no environment key)" % (extracted_val))
else:
extracted_val = env_value
LOG.debug("Using enviroment provided value %r" % (extracted_val))
else:
extracted_val = value
LOG.debug("Using raw config provided value %r" % (extracted_val))
return extracted_val
def no_remove(self):
if self._no_remove is not None:
return self._no_remove
packages = env.get_key('REQUIRED_PACKAGES', default_value='').split()
own_details = pip_helper.get_directory_details(os.getcwd())
required_pips = own_details['dependencies']
no_remove = self._convert_names_python2rpm(required_pips)
no_remove.extend(packages)
self._no_remove = no_remove
return self._no_remove
def package_start(self):
super(VenvDependencyHandler, self).package_start()
self.install_counters.clear()
base_cmd = env.get_key('VENV_CMD', default_value='virtualenv')
for instance in self.instances:
if not self._is_buildable(instance):
continue
# Create a virtualenv...
venv_dir = self._venv_directory_for(instance)
sh.mkdirslist(venv_dir, tracewriter=self.tracewriter)
cmd = [base_cmd, '--clear', venv_dir]
LOG.info("Creating virtualenv at %s", colorizer.quote(venv_dir))
out_filename = sh.joinpths(self.log_dir, "venv-create-%s.log" % (instance.name))
sh.execute_save_output(cmd, out_filename)
self._install_into_venv(instance,
self.PREREQUISITE_UPGRADE_PKGS,
upgrade=True)
def package_start(self):
super(VenvDependencyHandler, self).package_start()
self.install_counters.clear()
base_cmd = env.get_key('VENV_CMD', default_value='virtualenv')
for instance in self.instances:
if not self._is_buildable(instance):
continue
# Create a virtualenv...
venv_dir = self._venv_directory_for(instance)
sh.mkdirslist(venv_dir, tracewriter=self.tracewriter)
cmd = [base_cmd, '--clear', venv_dir]
LOG.info("Creating virtualenv at %s", colorizer.quote(venv_dir))
out_filename = sh.joinpths(self.log_dir,
"venv-create-%s.log" % (instance.name))
sh.execute_save_output(cmd, out_filename)
self._install_into_venv(instance,
self.PREREQUISITE_UPGRADE_PKGS,
upgrade=True)
def _get_default_dir():
root_dir = env.get_key('INSTALL_ROOT')
if root_dir:
return root_dir
return sh.joinpths(sh.gethomedir(), 'openstack')
def is_dry_run():
dry_v = env.get_key('ANVIL_DRYRUN')
if not dry_v:
return False
return tu.make_bool(dry_v)
def is_dry_run():
# Not stashed locally since the main entrypoint
# actually adjusts this value depending on a command
# line option...
return tu.make_bool(env.get_key('ANVIL_DRYRUN'))
import distutils.spawn
import psutil
import anvil
from anvil import env
from anvil import exceptions as excp
from anvil import log as logging
LOG = logging.getLogger(__name__)
# Locally stash these so that they can not be changed
# by others after this is first fetched...
SUDO_UID = env.get_key('SUDO_UID')
SUDO_GID = env.get_key('SUDO_GID')
# Tail this many lines from a file when executed and output is being
# piped to said file.
_TRUNCATED_OUTPUT_LINES = 7
# Take over some functions directly from os.path/os/... so that we don't have
# to type as many long function names to access these.
getsize = os.path.getsize
exists = os.path.exists
basename = os.path.basename
dirname = os.path.dirname
canon_path = os.path.realpath
prompt = raw_input
isfile = os.path.isfile
def is_dry_run():
dry_v = env.get_key('ANVIL_DRYRUN')
if not dry_v:
return False
return tu.make_bool(dry_v)
def run(args):
"""
Starts the execution after args have been parsed and logging has been setup.
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
try:
runner_cls = actions.class_for(action)
except Exception as ex:
raise excp.OptionException(str(ex))
if runner_cls.needs_sudo:
ensure_perms()
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" % (persona_fn))
# Determine + setup the root directory...
# If not provided attempt to locate it via the environment control files
args_root_dir = args.pop("dir")
root_dir = env.get_key('INSTALL_ROOT')
if not root_dir:
root_dir = args_root_dir
if not root_dir:
root_dir = sh.joinpths(sh.gethomedir(), 'openstack')
root_dir = sh.abspth(root_dir)
(repeat_string, line_max_len) = utils.welcome()
print(center_text("Action Runner", repeat_string, line_max_len))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Stash the dryrun value (if any)
if 'dryrun' in args:
sh.set_dry_run(args['dryrun'])
# Ensure the anvil dirs are there if others are about to use it...
ensure_anvil_dirs(root_dir)
# Load the distro
dist = distro.load(settings.DISTRO_DIR)
# Load + verify the person
try:
persona_obj = persona.load(persona_fn)
persona_obj.verify(dist)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" % (persona_fn, e))
# Get the object we will be running with...
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']), colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
def color_enabled():
if 'LOG_COLOR' in env.get() and not tu.make_bool(env.get_key('LOG_COLOR')):
return False
if not sys.stdout.isatty():
return False
return True
def run(args):
"""
Starts the execution after args have been parsed and logging has been setup.
Arguments: N/A
Returns: True for success to run, False for failure to start
"""
LOG.debug("CLI arguments are:")
utils.log_object(args, logger=LOG, level=logging.DEBUG, item_max_len=128)
# Keep the old args around so we have the full set to write out
saved_args = dict(args)
action = args.pop("action", '').strip().lower()
if action not in actions.names():
raise excp.OptionException("Invalid action name %r specified!" %
(action))
# Determine + setup the root directory...
# If not provided attempt to locate it via the environment control files
args_root_dir = args.pop("dir")
root_dir = env.get_key('INSTALL_ROOT')
if not root_dir:
root_dir = args_root_dir
if not root_dir:
root_dir = sh.joinpths(sh.gethomedir(), 'openstack')
root_dir = sh.abspth(root_dir)
sh.mkdir(root_dir)
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" %
(persona_fn))
# !!
# Here on out we should be using the logger (and not print)!!
# !!
# Stash the dryrun value (if any)
if 'dryrun' in args:
env.set("ANVIL_DRYRUN", str(args['dryrun']))
# Ensure the anvil etc dir is there if others are about to use it
ensure_anvil_dir()
# Load the distro
dist = distro.load(settings.DISTRO_DIR)
# Load + verify the person
try:
persona_obj = persona.load(persona_fn)
persona_obj.verify(dist)
except Exception as e:
raise excp.OptionException("Error loading persona file: %s due to %s" %
(persona_fn, e))
# Get the object we will be running with...
runner_cls = actions.class_for(action)
runner = runner_cls(distro=dist,
root_dir=root_dir,
name=action,
cli_opts=args)
(repeat_string, line_max_len) = utils.welcome()
print(center_text("Action Runner", repeat_string, line_max_len))
# Now that the settings are known to work, store them for next run
store_current_settings(saved_args)
LOG.info("Starting action %s on %s for distro: %s",
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()
runner.run(persona_obj)
end_time = time.time()
pretty_time = utils.format_time(end_time - start_time)
LOG.info("It took %s seconds or %s minutes to complete action %s.",
colorizer.quote(pretty_time['seconds']),
colorizer.quote(pretty_time['minutes']), colorizer.quote(action))
import subprocess
import sys
import time
import psutil # http://code.google.com/p/psutil/wiki/Documentation
import anvil
from anvil import env
from anvil import exceptions as excp
from anvil import log as logging
LOG = logging.getLogger(__name__)
# Locally stash these so that they can not be changed
# by others after this is first fetched...
SUDO_UID = env.get_key("SUDO_UID")
SUDO_GID = env.get_key("SUDO_GID")
# Set only once
IS_DRYRUN = None
# Take over some functions directly from os.path
getsize = os.path.getsize
class Process(psutil.Process):
def __str__(self):
return "%s (%s)" % (self.pid, self.name)
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
import termcolor
from anvil import env
from anvil import type_utils as tu
COLORS = termcolor.COLORS.keys()
LOG_COLOR = True
if 'LOG_COLOR' in env.get():
LOG_COLOR = tu.make_bool(env.get_key('LOG_COLOR'))
if not sys.stdout.isatty():
LOG_COLOR = False
def color_enabled():
return LOG_COLOR
def quote(data, quote_color='green', **kargs):
if not color_enabled():
return "'%s'" % (data)
else:
text = str(data)
if len(text) == 0:
text = "''"
def _get_default_dir():
root_dir = env.get_key('INSTALL_ROOT')
if root_dir:
return root_dir
return sh.joinpths(sh.gethomedir(), 'openstack')
def get(self, section, option):
return env.get_key(self._form_key(section, option))
import time
import distutils.spawn
import psutil
import anvil
from anvil import env
from anvil import exceptions as excp
from anvil import log as logging
LOG = logging.getLogger(__name__)
# Locally stash these so that they can not be changed
# by others after this is first fetched...
SUDO_UID = env.get_key('SUDO_UID')
SUDO_GID = env.get_key('SUDO_GID')
# Tail this many lines from a file when executed and output is being
# piped to said file.
_TRUNCATED_OUTPUT_LINES = 7
# Take over some functions directly from os.path/os/... so that we don't have
# to type as many long function names to access these.
getsize = os.path.getsize
exists = os.path.exists
basename = os.path.basename
dirname = os.path.dirname
canon_path = os.path.realpath
prompt = raw_input
isfile = os.path.isfile
def _get_default_dir():
root_dir = env.get_key("INSTALL_ROOT")
if root_dir:
return root_dir
return sh.joinpths(sh.gethomedir(), "openstack")