def set(self, shell=None, ptask=None):
# unset any previous custom env vars
self._unset_custom_vars(shell=shell)
# cd into the ptask directory
os.chdir(self.path)
if shell:
print shell.cd(self.path)
# check for config file
self._process_config(shell=shell, ptask=ptask)
# set the appropriate environment variables
self.env.set(shell=shell)
# add this spec to the history
if ptask:
if self.version:
spec = self.spec + PTaskSpec.VERSION + self.version
PTaskHistory().add(spec)
else:
PTaskHistory().add(self.spec)
# set the shell prompt if preferred
if shell:
ptask_prompt = DpaVars.ptask_prompt().get()
no_ptask_prompt = DpaVars.no_ptask_prompt().get()
if ptask and ptask_prompt:
print shell.set_prompt(ptask_prompt)
elif no_ptask_prompt:
print shell.set_prompt(no_ptask_prompt)
def set(self, shell=None, ptask=None):
# unset any previous custom env vars
self._unset_custom_vars(shell=shell)
# cd into the ptask directory
os.chdir(self.path)
if shell:
print shell.cd(self.path)
# check for config file
self._process_config(shell=shell, ptask=ptask)
# set the appropriate environment variables
self.env.set(shell=shell)
# add this spec to the history
if ptask:
if self.version:
spec = self.spec + PTaskSpec.VERSION + self.version
PTaskHistory().add(spec)
else:
PTaskHistory().add(self.spec)
# set the shell prompt if preferred
if shell:
ptask_prompt = DpaVars.ptask_prompt().get()
no_ptask_prompt = DpaVars.no_ptask_prompt().get()
if ptask and ptask_prompt:
print shell.set_prompt(ptask_prompt)
elif no_ptask_prompt:
print shell.set_prompt(no_ptask_prompt)
def __init__(self, history_file=None, history_size=None):
if history_file is None:
history_file = DpaVars.ptask_history_file().get()
if history_size is None:
history_size = DpaVars.ptask_history_size().get()
self.history_file = os.path.expanduser(history_file)
self.history_size = history_size
def __init__(self, history_file=None, history_size=None):
if history_file is None:
history_file = DpaVars.ptask_history_file().get()
if history_size is None:
history_size = DpaVars.ptask_history_size().get()
self.history_file = os.path.expanduser(history_file)
self.history_size = history_size
def __init__(self):
super(PTaskEnv, self).__init__()
# XXX document the accessor names
# library path
self.add(
DpaVars.path(default=DpaVars.path_base().get()),
name='path'
)
self.add(
DpaVars.ld_library_path(
default=DpaVars.ld_library_path_base().get()),
name='ld_library_path'
)
self.add(
DpaVars.python_path(default=DpaVars.python_path_base().get()),
name='python_path'
)
self.add(DpaVars.ptask_spec(), name='ptask_spec')
self.add(DpaVars.ptask_path(), name='ptask_path')
self.add(DpaVars.ptask_version(), name='ptask_version')
def __init__(self, spec, validate=True, version=None):
super(PTaskArea, self).__init__()
# should be a fully qualified ptask spec
if not isinstance(spec, PTaskSpec):
spec = PTaskSpec.get(spec)
ver_sep = PTaskSpec.VERSION
if ver_sep in spec:
if spec.count(ver_sep) > 1:
raise PTaskAreaError("Multiple version specs unsupported.")
(spec, version) = spec.split(ver_sep)
spec = PTaskSpec.get(spec)
# XXX this is making assumptions about a 1:1 correspondence
# between filesystem directories and ptask specs. this may not always
# be the case. need to consider cases where the ptasks in the hierarchy
# are spread across production disks for various reasons. at least this
# is isolated to this class and the API should remain the same.
self._spec = spec
self._version = version
self._base_spec = spec.base_spec
self._product_spec = spec.product_spec
if self._version and self._product_spec:
raise PTaskAreaError(
"PTask version not supported with product area initialization."
)
self._fs_root = DpaVars.filesystem_root().get()
self._root = DpaVars.projects_root().get()
self._base = os.path.join(*spec.split(PTaskSpec.SEPARATOR))
if self._version:
self._base = os.path.join(self._base, '.' + str(version).zfill(4))
self._path = os.path.join(self._root, self._base)
self._ancestor_paths = None
if validate and not self.exists():
raise PTaskAreaError("PTaskArea does not exist.")
def __init__(self, spec, validate=True, version=None):
super(PTaskArea, self).__init__()
# should be a fully qualified ptask spec
if not isinstance(spec, PTaskSpec):
spec = PTaskSpec.get(spec)
ver_sep = PTaskSpec.VERSION
if ver_sep in spec:
if spec.count(ver_sep) > 1:
raise PTaskAreaError("Multiple version specs unsupported.")
(spec, version) = spec.split(ver_sep)
spec = PTaskSpec.get(spec)
# XXX this is making assumptions about a 1:1 correspondence
# between filesystem directories and ptask specs. this may not always
# be the case. need to consider cases where the ptasks in the hierarchy
# are spread across production disks for various reasons. at least this
# is isolated to this class and the API should remain the same.
self._spec = spec
self._version = version
self._base_spec = spec.base_spec
self._product_spec = spec.product_spec
if self._version and self._product_spec:
raise PTaskAreaError(
"PTask version not supported with product area initialization.")
self._fs_root = DpaVars.filesystem_root().get()
self._root = DpaVars.projects_root().get()
self._base = os.path.join(*spec.split(PTaskSpec.SEPARATOR))
if self._version:
self._base = os.path.join(self._base, '.' + str(version).zfill(4))
self._path = os.path.join(self._root, self._base)
self._ancestor_paths = None
if validate and not self.exists():
raise PTaskAreaError("PTaskArea does not exist.")
def from_directory(cls, directory):
"""Determine a spec from a local directory."""
projects_root = DpaVars.projects_root().get()
spec = directory.strip()
spec = spec.replace(projects_root, "")
spec = spec.replace(os.path.sep, cls.SEPARATOR)
spec = spec.strip(cls.SEPARATOR)
return spec
def from_directory(cls, directory):
"""Determine a spec from a local directory."""
projects_root = DpaVars.projects_root().get()
spec = directory.strip()
spec = spec.replace(projects_root, "")
spec = spec.replace(os.path.sep, cls.SEPARATOR)
spec = spec.strip(cls.SEPARATOR)
return spec
def current_location_code():
"""Retrieve the code of the current location.
:rtype: str
:return: The code of the current location.
>>> from dpa.location import current_location_code
>>> code = current_location_code()
>>> print code
'CU_MCADAMS_DPA'
"""
return DpaVars.location_code("").get()
def current_location_code():
"""Retrieve the code of the current location.
:rtype: str
:return: The code of the current location.
>>> from dpa.location import current_location_code
>>> code = current_location_code()
>>> print code
'CU_MCADAMS_DPA'
"""
return DpaVars.location_code("").get()
def _unset_custom_vars(self, shell=None):
try:
custom_vars = self.env.custom_vars
except EnvError:
custom_vars = DpaVars.ptask_custom_vars()
custom_vars.get()
for var_name in custom_vars.list:
if var_name:
env_var = EnvVar(var_name)
env_var.unset(shell=shell)
# unset the custom vars variable itself
custom_vars.unset(shell=shell)
def _unset_custom_vars(self, shell=None):
try:
custom_vars = self.env.custom_vars
except EnvError:
custom_vars = DpaVars.ptask_custom_vars()
custom_vars.get()
for var_name in custom_vars.list:
if var_name:
env_var = EnvVar(var_name)
env_var.unset(shell=shell)
# unset the custom vars variable itself
custom_vars.unset(shell=shell)
def __init__(self):
super(PTaskEnv, self).__init__()
# XXX document the accessor names
# library path
self.add(DpaVars.path(default=DpaVars.path_base().get()), name='path')
self.add(DpaVars.ld_library_path(
default=DpaVars.ld_library_path_base().get()),
name='ld_library_path')
self.add(DpaVars.python_path(default=DpaVars.python_path_base().get()),
name='python_path')
self.add(DpaVars.ptask_spec(), name='ptask_spec')
self.add(DpaVars.ptask_path(), name='ptask_path')
self.add(DpaVars.ptask_version(), name='ptask_version')
def validate(self):
cur_spec = PTaskArea.current().spec
full_spec = PTaskSpec.get(self.spec, relative_to=cur_spec)
# if we're listing the current ptask's subs, and no versions specified
if cur_spec == full_spec and not self._versions:
ptask_ver = DpaVars.ptask_version().get()
if ptask_ver:
self._versions = [ptask_ver]
if not self._versions:
self._versions = ["latest"]
# try to get a ptask instance from the db
try:
ptask = PTask.get(full_spec)
except PTaskError as e:
# fall back to the input spec
try:
ptask = PTask.get(self.spec)
except PTaskError:
raise ActionError(
'Could not determine ptask from: "{s}"'.format(
s=self.spec)
)
self._ptask = ptask
if self._versions == ['latest']:
versions = [self.ptask.latest_version]
elif self._versions == ['all']:
versions = self.ptask.versions
else:
self._versions = map(int, self._versions)
versions = [v for v in self.ptask.versions
if v.number in self._versions]
if len(versions) == 0:
raise ActionError(
"No matches found for {p} version: {v}".format(
p=ptask.spec,
v=Style.bright + str(self._versions) + Style.normal,
)
)
self._versions = versions
def validate(self):
cur_spec = PTaskArea.current().spec
full_spec = PTaskSpec.get(self.spec, relative_to=cur_spec)
# if we're listing the current ptask's subs, and no versions specified
if cur_spec == full_spec and not self._versions:
ptask_ver = DpaVars.ptask_version().get()
if ptask_ver:
self._versions = [ptask_ver]
if not self._versions:
self._versions = ["latest"]
# try to get a ptask instance from the db
try:
ptask = PTask.get(full_spec)
except PTaskError as e:
# fall back to the input spec
try:
ptask = PTask.get(self.spec)
except PTaskError:
raise ActionError(
'Could not determine ptask from: "{s}"'.format(
s=self.spec))
self._ptask = ptask
if self._versions == ['latest']:
versions = [self.ptask.latest_version]
elif self._versions == ['all']:
versions = self.ptask.versions
else:
self._versions = map(int, self._versions)
versions = [
v for v in self.ptask.versions if v.number in self._versions
]
if len(versions) == 0:
raise ActionError("No matches found for {p} version: {v}".format(
p=ptask.spec,
v=Style.bright + str(self._versions) + Style.normal,
))
self._versions = versions
def _validate_location(self):
# ---- make sure code and server are valid
# first, set the server value in the environment
server_var = DpaVars.data_server()
server_var.value = self.server
server_var.set()
# now query the location code
try:
location = Location.get(self.code)
except ActionError as e:
raise ActionError(
"Unable to verify location: " + self.code + "\n" + str(e)
)
return location
def validate(self):
use_cur_version = False
if not isinstance(self.ptask, PTask):
if not self.ptask or self.ptask == '.':
use_cur_version = True
cur_spec = PTaskArea.current().spec
full_spec = PTaskSpec.get(self.ptask, relative_to=cur_spec)
# try to get a ptask instance from the db
try:
ptask = PTask.get(full_spec)
except PTaskError as e:
# fall back to the input spec
try:
ptask = PTask.get(self.spec)
except PTaskError:
raise ActionError(
'Could not determine ptask from: "{s}"'.format(
s=self.spec)
)
self._ptask = ptask
latest_ver = self.ptask.latest_version
if use_cur_version:
cur_ptask_ver = DpaVars.ptask_version().get()
if cur_ptask_ver and cur_ptask_ver != latest_ver.number:
self._ptask_version = self.ptask.version(cur_ptask_ver)
self._version = cur_ptask_ver
else:
self._ptask_version = self.ptask.latest_version
else:
self._ptask_version = self.ptask.latest_version
def require_executable(self, executable):
"""Returns the full path for the supplied executable name."""
(path, file_name) = os.path.split(executable)
# path already included
if path:
if not os.path.isfile(executable):
raise SessionError("Unable to locate executable: " + executable)
elif not os.access(executable, os.X_OK):
raise SessionError("File is not executable: " + executable)
else:
return executable
else:
bin_paths = DpaVars.path()
bin_paths.get()
for path in bin_paths.list:
executable_path = os.path.join(path, executable)
if (os.path.isfile(executable_path) and
os.access(executable_path, os.X_OK)):
return executable_path
raise SessionError("Unable to locate executable: " + executable)
def validate(self):
use_cur_version = False
if not isinstance(self.ptask, PTask):
if not self.ptask or self.ptask == '.':
use_cur_version = True
cur_spec = PTaskArea.current().spec
full_spec = PTaskSpec.get(self.ptask, relative_to=cur_spec)
# try to get a ptask instance from the db
try:
ptask = PTask.get(full_spec)
except PTaskError as e:
# fall back to the input spec
try:
ptask = PTask.get(self.spec)
except PTaskError:
raise ActionError(
'Could not determine ptask from: "{s}"'.format(
s=self.spec))
self._ptask = ptask
latest_ver = self.ptask.latest_version
if use_cur_version:
cur_ptask_ver = DpaVars.ptask_version().get()
if cur_ptask_ver and cur_ptask_ver != latest_ver.number:
self._ptask_version = self.ptask.version(cur_ptask_ver)
self._version = cur_ptask_ver
else:
self._ptask_version = self.ptask.latest_version
else:
self._ptask_version = self.ptask.latest_version
def _process_config(self, shell=None, ptask=None):
# get all ancestor ptask config files
ancestor_paths = self.ancestor_paths()
filename = self.__class__._PTASK_SET_CONFIG
# reverse order for compositing
config_files = reversed(
[os.path.join(p, filename) for p in ancestor_paths])
config = Config.composite(config_files)
for (key, value) in config.iteritems():
# echo to shell
if shell and key.startswith('echo'):
try:
value = value.format(
ptask=ptask,
style=Style,
bg=Bg,
fg=Fg,
)
print shell.echo(value)
except:
pass
# run command
if shell and key.startswith("cmd"):
for cmd in value:
try:
cmd = cmd.format(ptask=ptask)
print shell.command(cmd)
except Exception as e:
print shell.echo("ERROR: " + str(e))
# change directory
elif shell and key == "cd":
print shell.cd(value)
# set env var
elif key.startswith("env"):
# the value is another config object with env vars
for (var_key, var_value) in value.iteritems():
try:
custom_vars = self.env.custom_vars
except:
# add a variable that remembers per-ptask custom
# variables. we'll use this to know what to unset when
# setting a new ptask.
self.env.add(DpaVars.ptask_custom_vars(),
name='custom_vars')
var = EnvVar(var_key)
var.value = var_value
var.set(shell=shell)
# remember this variable name for unsetting
if not var_key in self.env.custom_vars.list:
self.env.custom_vars.append(var_key)
class Logger(object):
# ------------------------------------------------------------------------
# Public class attributes:
# ------------------------------------------------------------------------
# the available levels for logging
levels = ['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
# log to the shared log directory, or a tempdirectory if undefined
log_dir = DpaVars.share_logs(default=tempfile.gettempdir()).get()
# the default logging level
log_level = "WARNING"
# ------------------------------------------------------------------------
# Private class attributes:
# ------------------------------------------------------------------------
# ---- setup consistent stdout handling for all dpa code
# formatting for all logging to stdout
_stdout_formatter = logging.Formatter(
fmt="%(name)s %(levelname)s: %(message)s")
# handler for all stdout. Anything logged at a level matching or above
# should go to stdout
_stdout_handler = logging.StreamHandler()
_stdout_handler.setFormatter(_stdout_formatter)
_stdout_handler.setLevel(_log_level_from_name(log_level))
# format all log messages
_logfile_formatter = logging.Formatter(
fmt="[%(asctime)s] {un}@{ma} %(name)s %(levelname)s: %(message)s".\
format(ma=platform.node(), un=current_username()),
datefmt="%Y/%m/%d-%H:%M:%S",
)
# attach the handler to the root logger.
_root_logger = logging.getLogger()
_root_logger.addHandler(_stdout_handler)
# keep track of processed logger names
_logger_cache = dict()
# ------------------------------------------------------------------------
# Class methods:
# ------------------------------------------------------------------------
@classmethod
def get(cls, name=None):
if not name:
name = "dpa"
# make sure everything is namespaced with dpa.
if not name.startswith("dpa"):
name = ".".join(["dpa", name])
if not name in cls._logger_cache.keys():
# create the custom logger wrapper
# log to a file based on the logger's name
log_file = name + ".log"
log_path = os.path.join(cls.log_dir, log_file)
# ensure path exists with proper permissions first
if not os.path.exists(log_path):
_create_log_file(log_path)
# create a handler to output to a log file, set the level to INFO
logfile_handler = logging.FileHandler(filename=log_path)
logfile_handler.setLevel(_log_level_from_name('INFO'))
logfile_handler.setFormatter(cls._logfile_formatter)
# get the named logger, set have it process all messages, and
# have it send output to the file handler. note the level is
# set on the handler.
logger = logging.getLogger(name)
logger.setLevel(_log_level_from_name('DEBUG'))
logger.addHandler(logfile_handler)
cls._logger_cache[name] = logger
return cls._logger_cache[name]
# ------------------------------------------------------------------------
@classmethod
def set_level(cls, level):
"""Set the level for the root logger's stdout."""
cls.log_level = level
cls._stdout_handler.setLevel(_log_level_from_name(level))
def __init__(self, data_server=None):
if data_server is None:
from dpa.env.vars import DpaVars
self.data_server = DpaVars.data_server().get()
else:
self.data_server = data_server
def _process_config(self, shell=None, ptask=None):
# get all ancestor ptask config files
ancestor_paths = self.ancestor_paths()
filename = self.__class__._PTASK_SET_CONFIG
# reverse order for compositing
config_files = reversed(
[os.path.join(p, filename) for p in ancestor_paths])
config = Config.composite(config_files)
for (key, value) in config.iteritems():
# echo to shell
if shell and key.startswith('echo'):
try:
value = value.format(
ptask=ptask,
style=Style,
bg=Bg,
fg=Fg,
)
print shell.echo(value)
except:
pass
# run command
if shell and key.startswith("cmd"):
for cmd in value:
try:
cmd = cmd.format(ptask=ptask)
print shell.command(cmd)
except Exception as e:
print shell.echo("ERROR: " + str(e))
# change directory
elif shell and key == "cd":
print shell.cd(value)
# set env var
elif key.startswith("env"):
# the value is another config object with env vars
for (var_key, var_value) in value.iteritems():
try:
custom_vars = self.env.custom_vars
except:
# add a variable that remembers per-ptask custom
# variables. we'll use this to know what to unset when
# setting a new ptask.
self.env.add(
DpaVars.ptask_custom_vars(), name='custom_vars')
var = EnvVar(var_key)
var.value = var_value
var.set(shell=shell)
# remember this variable name for unsetting
if not var_key in self.env.custom_vars.list:
self.env.custom_vars.append(var_key)
def __init__(self, data_server=None):
if data_server is None:
from dpa.env.vars import DpaVars
self.data_server = DpaVars.data_server().get()
else:
self.data_server = data_server