class PyPLearnScript( PyPLearnObject ):
"""Feeded by the PyPLearn driver to PLearn's main.
This class has a PLearn cousin of the same name that is used to wrap
PyPLearn scripts feeded to PLearn application. Most user need not to
care about the behaviour of this class and its cousin since their
effects are hidden in the core of the PLearn main program.
Basically, feeding a PyPLearnScript to PLearn allows management of
files to be writen in the experiment directory after the experiment was
ran.
"""
expdir = PLOption(None)
metainfos = PLOption(None)
plearn_script = PLOption(None)
def __init__(self, main_object, **overrides):
PyPLearnObject.__init__(self, **overrides)
self.expdir = plargs.expdir
self.plearn_script = str( main_object )
self.metainfos = self.getMetainfos()
def getMetainfos(self):
bindings = plargs.getContextBindings()
def backward_cast(value):
if isinstance(value, list):
return ",".join([ str(e) for e in value ])
return str(value)
pretty = lambda opt, val: (opt.ljust(45), backward_cast(val))
cmdline = [ "%s = %s"%pretty(opt, bindings[opt]) for opt in bindings ]
return "\n".join(cmdline)
class ModeParsingOptions(PyPLearnObject):
usage = PLOption(None)
option_list = PLOption(None)
# option_class = Option
version = PLOption(None)
conflict_handler = PLOption("error")
description = PLOption(None)
formatter = PLOption(None)
add_help_option = PLOption(True)
prog = PLOption(None)
class Test(PyPLearnObject):
opt1 = PLOption("option1")
opt2 = PLOption(2)
opt3 = PLOption({"3":3})
class Subclass(PyPLearnObject):
option1 = PLOption("opt1")
option2 = PLOption(2)
class Test(PyPLearnObject):
option1 = PLOption("value1")
option2 = PLOption(2)
option3 = PLOption({"key3": 3})
class Program(core.PyTestObject):
####### Options #############################################################
name = PLOption(None)
compiler = PLOption(None)
compile_options = PLOption(None)
no_plearn_options = PLOption(
False,
doc=
"PLearn commands usually receive the --no-progess and --no-version options. "
"If that option is False though, this won't be the case for this Program instance."
)
dependencies = PLOption(
[], doc="A list of programs on which this one depends.")
####### Class Variables #####################################################
# Default compiler: for programs assumed to be compilable but for which
# no compiler were provided
default_compiler = "pymake"
# Cache to remember which executable already exist
compiled_programs = {}
# If True, no compilations done; assume old executables are valid
compilation_disabled = False
# Special pymake hack --- see handleCompileOptions()
cmdline_compile_options = ""
####### Instance Methods ####################################################
def __init__(self, **overrides):
core.PyTestObject.__init__(self, **overrides)
assert self.name is not None
# Some command line options can modify the default values for those...
self.handleCompileOptions()
# Methods are called even if messages are not logged...
## The getProgramPath() method call is required here so as to
## initialize the '_program_path' and '__is_global' members
logging.debug("\nProgram: " + self.getProgramPath())
internal_exec_path = self.getInternalExecPath(self._signature())
logging.debug("Internal Exec Path: %s" % internal_exec_path)
if self.isCompilable():
if self.compiler is None:
self.compiler = Program.default_compiler
self.__attempted_to_compile = False
self.__log_file_path = self.getCompilationLogPath()
logging.debug("Program instance: %s\n" % repr(self))
def handleCompileOptions(self):
assert self.compiler is not None or self.compile_options is None, \
"One cannot provide compile options without providing a compiler..."
logging.debug(
"Creating program using compiler %s with compile_options '%s'." %
(self.compiler, self.compile_options))
if self.cmdline_compile_options and self.compiler == "pymake":
config_options = []
if self.compile_options:
config_options = [
opt.replace("-", "")
for opt in self.compile_options.split()
]
cmdline_options = self.cmdline_compile_options.split(",")
options = list(set(config_options + cmdline_options))
options = " -".join([""] + options).strip()
logging.debug(
"Test %s: Using compile options '%s' instead of '%s'...",
self.name, self.compile_options, options)
self.compile_options = options
def _signature(self):
if self.compiler is None:
signature = self.name
else:
if self.compile_options == "":
self.compile_options = None
signature = "%s__compiler_%s__options_%s" % (
self.name, self.compiler, self.compile_options)
signature = signature.replace('-', '') # Compile options...
return signature.replace(' ', '_')
def _optionFormat(self, option_pair, indent_level, inner_repr):
optname, val = option_pair
if val is None:
return ""
elif optname == "no_plearn_options" and not val:
return "" # Don't print the default value
elif optname == "dependencies" and not val:
return "" # Don't print the default value
else:
return super(Program, self)._optionFormat(option_pair,
indent_level, inner_repr)
def compilationSucceeded(self):
exec_exists = os.path.exists(self.getInternalExecPath())
no_need_to_compile = self.compilation_disabled or not self.isCompilable(
)
if no_need_to_compile and not exec_exists:
### NOTE: This could be changed by a 'cp getProgramPath() getInternalExecPath()'
raise core.PyTestUsageError(
"Called PyTest with --no-compile option but %s "
"was not previously compiled." % self.getInternalExecPath())
# Account for dependencies
success = no_need_to_compile or (self.__attempted_to_compile
and exec_exists)
#logging.debug("compilationSucceeded(): %s %s %s %s %s", self.getInternalExecPath(),#self.name,
# success, no_need_to_compile, self.__attempted_to_compile, exec_exists)
for dep in self.dependencies:
success = (success and dep.compilationSucceeded())
#print "+++ DEP compilationSucceeded():", success
return success
def compile(self, publish_dirpath=""):
succeeded = True
if self.isCompilable():
# Remove old compile log if any
publish_target = os.path.join(
publish_dirpath, os.path.basename(self.__log_file_path))
if os.path.islink(publish_target) or os.path.isfile(
publish_target):
os.remove(publish_target)
assert not os.path.exists(publish_target)
# Ensure compilation is needed
if self.compilationSucceeded():
logging.debug("Already successfully compiled %s" %
self.getInternalExecPath())
succeeded = True
elif self.__attempted_to_compile:
logging.debug("Already attempted to compile %s" %
self.getInternalExecPath())
succeeded = False
# First compilation attempt
else:
#print "+++ SHORTCUT!!!", self.name
#succeeded = True
succeeded = self.__first_compilation_attempt()
#print "+++ FIRST ATTEMPT", self.name, succeeded
# Publish the compile log
if succeeded and publish_dirpath:
logging.debug("Publishing the compile log %s" %
self.__log_file_path)
toolkit.symlink(self.__log_file_path,
moresh.relative_path(publish_target))
# Account for dependencies
#print "+++ Success", self.name, succeeded
for dep in self.dependencies:
succeeded = (succeeded and dep.compile(publish_dirpath))
#print "+++ DEPcompile", succeeded
return succeeded
def __first_compilation_attempt(self):
####### First compilation attempt ####################################
targetdir, exec_name = os.path.split(self.getInternalExecPath())
sanity_check, log_fname = os.path.split(self.__log_file_path)
assert sanity_check == targetdir
if sys.platform == 'win32':
# When there are characters like '=' in a filenme, Windows can get
# confused if the filename is not quoted.
log_fname = '"%s"' % log_fname
# Remove outdated log file
assert not os.path.exists(exec_name)
if os.path.exists(log_fname):
os.remove(log_fname)
elif not os.path.exists(targetdir):
os.makedirs(targetdir)
# Actual compilation
moresh.pushd(targetdir)
logging.debug("\nIn %s:" % os.getcwd())
if sys.platform == 'win32':
# We assume the compiler is pymake.
if self.compiler != "pymake":
raise Not_Implemented
the_compiler = "python " + \
os.path.join(ppath.ppath('PLEARNDIR'), 'scripts', 'pymake')
else:
the_compiler = self.compiler
compile_options = ""
if self.compile_options is not None:
compile_options = self.compile_options
compile_cmd = self.getCompileCommand(the_compiler, compile_options)
logging.debug(compile_cmd)
p = subprocess.Popen(compile_cmd,
shell=True,
stdout=open(log_fname, 'w'),
stderr=subprocess.STDOUT)
compile_exit_code = p.wait()
logging.debug("compile_exit_code <- %d\n" % compile_exit_code)
moresh.popd()
# Report success of fail and remember that compilation was attempted
self.__attempted_to_compile = True
if compile_exit_code != 0 and os.path.exists(
self.getInternalExecPath()):
os.remove(self.getInternalExecPath())
# Strip C++ execs
if self.isCompilable() and self.compilationSucceeded():
os.system("strip %s" % self.getInternalExecPath())
return compile_exit_code == 0
def getCompileCommand(self, the_compiler, compile_options):
compile_cmd = "%s %s %s -link-target %s" \
% ( the_compiler, compile_options,
self.getProgramPath(),
self.getInternalExecPath())
return compile_cmd
def getName(self):
return self.name
def getCompilationLogPath(self):
return self.getInternalExecPath() + '.log'
def getInternalExecPath(self, candidate=None):
if hasattr(self, '_internal_exec_path'):
assert candidate is None, 'candidate is not None:' + repr(
candidate)
return self._internal_exec_path
logging.debug("Parsing for internal exec path; candidate=%s" %
candidate)
if candidate == self.name:
self._internal_exec_path = self.getProgramPath()
else:
self._internal_exec_path = \
os.path.join(core.pytest_config_path(), "compiled_programs", candidate)
if sys.platform == 'win32':
# Cygwin has trouble with the \ characters.
self._internal_exec_path = \
self._internal_exec_path.replace('\\', '/')
# In addition, we need to add the '.exe' extension as otherwise it
# may not be able to run it.
self._internal_exec_path += '.exe'
return self._internal_exec_path
def getProgramPath(self):
if hasattr(self, '_program_path'):
return self._program_path
try:
self._program_path = find_local_program(self.name)
self.__is_global = False
except core.PyTestUsageError, not_local:
try:
self._program_path = find_global_program(self.name)
self.__is_global = True
except core.PyTestUsageError, neither_global:
raise core.PyTestUsageError("%s %s" %
(not_local, neither_global))
class Experiment(PyPLearnObject):
##
# Options
path = PLOption(None)
expkey = PLOption(None)
abspath = PLOption(None)
##
# Class variables
_expdir_prefix = 'expdir'
_metainfos_fname = 'metainfos.txt'
_cached_exp_fname = 'Experiment.cache'
_lhs_length = 35
_cached = None # See cache_experiments
_opened_pmats = []
##
# PyPLearnObject's classmethod
_by_value = classmethod(lambda cls: True)
def cache_experiments(cls, exproot=None, forget=True, name_key=None):
if exproot is None:
roots = pyplearn.config.get_option('EXPERIMENTS',
'expdir_root').split(',')
for exproot in roots:
cls.cache_experiments(exproot=exproot,
forget=False,
name_key=name_key)
return
if cls._cached is None:
cls._cached = []
elif forget:
for exp in cls._cached:
exp.close()
cls._cached = []
if exproot:
if not os.path.exists(exproot):
return
dirlist = os.listdir(exproot)
else:
dirlist = os.listdir(os.getcwd())
for fname in dirlist:
candidate_path = os.path.join(exproot, fname)
candidate_infopath = os.path.join(candidate_path,
cls._metainfos_fname)
if fname.startswith( cls._expdir_prefix ) \
and os.path.exists(candidate_infopath):
x = cls(path=candidate_path)
if name_key:
x.setName(x.expkey[name_key])
cls._cached.append(x)
return cls._cached
cache_experiments = classmethod(cache_experiments)
def match(cls, expkey=[]):
if cls._cached is None:
cls.cache_experiments()
return [exp for exp in cls._cached if exp.isMatched(expkey)]
match = classmethod(match)
def match_one(cls, expkey):
matches = cls.match(expkey)
assert len(matches) == 1, \
"Key matches %d experiments\n%s" % (len(matches), expkey)
return matches[0]
match_one = classmethod(match_one)
#
# Instance methods
#
def __init__(self, **overrides):
PyPLearnObject.__init__(self, **overrides)
self._name = None
if self.expkey is None:
self.expkey = ExpKey(os.path.join(self.path,
self._metainfos_fname))
# Update abspath
self.abspath = os.path.abspath(self.path)
def __del__(self):
self.close()
def close(self):
for pmat in self._opened_pmats:
pmat.close()
def __cmp__(self, other):
raise NotImplementedError('Use keycmp( x1, x2, expkey )')
def __str__(self):
return self.toString()
def loadPMat(self, *pmats):
for pmat in pmats:
attr_name = os.path.basename(pmat).replace('.pmat', '')
if hasattr(self, attr_name):
assert getattr(self, "_pmat_%s" % attr_name) == pmat
else:
setattr(self, "_pmat_%s" % attr_name, pmat)
pmat = PMat(os.path.join(self.abspath, pmat))
setattr(self, attr_name, pmat)
self._opened_pmats.append(pmat)
return self._opened_pmats[-1]
def getKey(self, expkey=None):
if expkey is None:
return self.expkey
subset = ExpKey()
for key in expkey:
if key in self.expkey:
subset[key] = self.expkey[key]
else:
subset[key] = None
return subset
def isMatched(self, expkey):
# Always matching empty expkey
if not expkey:
return True
# User should probably become aware of the concept of ExpKey.
if not isinstance(expkey, ExpKey):
expkey = ExpKey(expkey)
# For efficiency
if len(expkey) > len(self.expkey):
return False
# A key element from the expkey matches this experiement if
#
# 1) the key exists within this experiment ExpKey
#
# 2) the value is not restricted (None) or is restricted the same
# value than the one in this experiment
match_predicate = lambda lhs,rhs: \
lhs in self.expkey and \
( rhs is None or self.expkey[lhs]==rhs )
# All key element must match (match_predicate)
for lhs, rhs in expkey.iteritems():
if not match_predicate(lhs, rhs):
return False
# All key element matched
return True
def toString(self, expkey=None, short=False):
s = '%s\n' % self.path
if short and expkey is None:
return s
for key, value in self.getKey(expkey).iteritems():
s += ' %s= %s\n' % (key.ljust(30), value)
return s
def running(self):
return len(self.expkey) == 0
### set/getName
def getName(self):
assert self._name
return self._name
def setName(self, name):
self._name = name
class Dispatch( PyPLearnObject ):
# The name of the program to invoke
program = PLOption(None)
# If provided, will always be the first argument
script = PLOption(str)
# These are args to be provided for each call to program in
# addition the args returned by the oracle. Must be a string or a
# list of strings.
constant_args = PLOption(list)
# Either "expkey" or "named_args"
protocol = PLOption("expkey")
max_nmachines = PLOption(6,
doc="Max number of machines. Use -1 for no limit.")
logdir = PLOption(None) #"LOGS"
# Path to the directory where experiments are to be loaded
expdir_root = PLOption(None) # Default: os.getcwd()
delay = PLOption(False)
## Time interval (seconds) to wait before launching a new task
launch_delay_seconds = PLOption(1)
allow_unexpected_options = lambda self : False
def __init__( self, oracles=None,
_predicate=inexistence_predicate, **overrides ):
PyPLearnObject.__init__( self, **overrides )
self._predicate = _predicate
# Append current path to PYTHONPATH
if self.expdir_root is None:
self.expdir_root = os.getcwd()
self.__check_constant_args()
# If oracles were already provided, the ctor can lauch the tasks...
if oracles:
self.start( *oracles )
def __check_constant_args( self ):
if isinstance( self.constant_args, str ):
self.constant_args = [ self.constant_args ]
assert isinstance( self.constant_args, list )
def getArguments( self, argument_bindings ):
"""Parses for special keywords and join keys to values.
Special keywords:
1) _program_ -> self.program
2) _script_ -> self.script
3) _constant_args_ -> self.constant_args
4) expdir_root -> Experiments are cached.
Keys and values are joined using the ARG_VALUE_FMT string.
"""
self.program = argument_bindings.pop( "_program_", self.program )
self.script = argument_bindings.pop( "_script_", self.script )
self.constant_args = argument_bindings.pop( "_constant_args_", self.constant_args )
self.__check_constant_args()
expdir_root = os.path.abspath( argument_bindings.get( "expdir_root", self.expdir_root ) )
if expdir_root != self.expdir_root:
self.expdir_root = expdir_root
if os.path.isdir(self.expdir_root):
Experiment.cache_experiments( self.expdir_root )
if self.protocol=="expkey":
expkey = [ ARG_VALUE_FMT%(k,v) for (k,v) in argument_bindings.iteritems() ]
if not self._predicate( expkey ):
raise RejectedByPredicate( ' '.join(QUOTED_ARGS(expkey)) )
return expkey
elif self.protocol=="named_args":
return [ arg%argument_bindings for arg in self.constant_args ]
def start( self, *oracles ):
"""Frees all tasks if a keyboard interrupt is caught."""
if self.logdir is not None:
if not os.path.exists(self.logdir):
os.mkdir(self.logdir)
# Module function set_logdir()
set_logdir(self.logdir)
try:
task_sum = 0
delayed_tasks = 0
for arguments_oracle in oracles:
assert isinstance( arguments_oracle, ArgumentsOracle ), TypeError(type(arguments_oracle))
done, delayed = self.__start( arguments_oracle )
task_sum += done
delayed_tasks += delayed
if task_sum:
logging.info("\n(%d tasks done)"%task_sum)
if delayed_tasks:
logging.info("\n(%d tasks delayed)"%delayed_tasks)
logging.info( "[On %d requested experiments.]"
% sum([ len(oracle) for oracle in oracles ]) )
except KeyboardInterrupt:
logging.error("! Interrupted by user.")
self.free()
def __start(self, arguments_oracle):
"""Parallel dispatch; respecting max_nmachines."""
counter = 0
delayed = 0
for argument_bindings in arguments_oracle:
try:
# Parses for special keywords and join keys to values.
arguments = self.getArguments( argument_bindings )
except RejectedByPredicate, rejected:
logging.info('Already exists: %s'%str(rejected))
continue
assert self.program
prepend = [ "echo", "$HOST;", NICE, self.program ]
if self.script:
prepend.append( self.script )
if self.protocol=="expkey":
prepend.extend( self.constant_args )
# No expdir is created if there is no script or if the script
# is not a pyplearn script
if self.script.find( '.pyplearn' ) != -1:
expdir = None
for arg in arguments:
if arg.startswith("expdir="):
expdir = arg
break
if expdir is None:
expdir = generateExpdir()
## Making sure the next expdir will be generated at
## another 'time', i.e. on another second
time.sleep(self.launch_delay_seconds)
arguments.append( "expdir=%s" % expdir )
# # Module function defined above
# launch_task( prepend+QUOTED_ARGS(arguments) )
assert Task is not None
##TBM?: task = Task(prepend+QUOTED_ARGS(arguments)+[";", "echo", "'Task Done.'"])
if self.delay:
##TBM?:
cmd = ' '.join(prepend+QUOTED_ARGS(arguments))
logging.info('Delayed: %s'%cmd)
##TBM?: logging.info('Delayed: %s'%task.getLaunchCommand())
delayed += 1
##TBM?: task.free() # Since it won't be launch, free the resources...
continue
##TBM?:
task = Task(prepend+QUOTED_ARGS(arguments)+[";", "echo", "'Task Done.'"])
task.launch()
if Task.count( ) == self.max_nmachines:
logging.info( "+++ Using %d machines or more; waiting..."
% self.max_nmachines )
completed = Task.select()
counter += 1
## Wait for all experiments completion
while Task is not None:
try:
completed = Task.select()
except EmptyTaskListError:
break
return counter, delayed
class VPrinter(PyPLearnObject):
"""Manages verbosity option for Python applications."""
##
# Class variables
_vlevels = {
'quiet': -1,
'normal': 0,
'verbose': 5,
'debug': 10,
}
##
# Options
box_width = PLOption(100)
is_global = PLOption(False)
output = PLOption(lambda: sys.stderr)
verbosity = PLOption('normal')
def __init__(self, **overrides):
super(VPrinter, self).__init__(**overrides)
self._vlevel = self.__class__._vlevels[self.verbosity]
self._indent = ''
self._indent_stack = []
if self.is_global:
globals()['__global_vprint'].setVPR(self)
def __call__(self, message='', priority='normal', highlight=''):
if isinstance(priority, str):
priority = self._vlevels[priority]
assert priority >= 0
if priority > self._vlevel:
return
if highlight:
assert len(highlight) == 1
self._print()
self._print(highlight * self.box_width)
self.indent(highlight + ' ')
lines = str(message).split('\n')
for line in lines:
for subline in box(line, self.box_width, self._indent,
' ' + highlight):
self._print(subline)
if highlight:
self.dedent()
self._print(highlight * self.box_width)
self._print()
def _print(self, s=''):
print >> self.output, s
def dedent(self):
self._indent = self._indent[:-self._indent_stack.pop()]
def indent(self, s=" " * 4):
self._indent += s
self._indent_stack.append(len(s))
def setOutput(self, output):
self.output = output