def load_bytecode_processors():
"""
Loading BytecodeProcessor from modipyd.BYTECODE_PROCESSORS
settings. Return ChainedBytecodeProcessor instance holds
all loaded processors.
"""
if (BYTECODE_PROCESSORS and
(len(BYTECODE_PROCESSORS_CACHE) != len(BYTECODE_PROCESSORS))):
del BYTECODE_PROCESSORS_CACHE[:]
for i, name in enumerate(BYTECODE_PROCESSORS[:]):
LOGGER.info("Loading BytecodeProcesser '%s'" % name)
try:
klass = utils.import_component(name)
except (ImportError, AttributeError):
LOGGER.warn(
"Loading BytecodeProcesser '%s' failed. "
"This setting is removed" % name,
exc_info=True)
del BYTECODE_PROCESSORS[i]
else:
BYTECODE_PROCESSORS_CACHE.append(klass)
processors = []
for klass in BYTECODE_PROCESSORS_CACHE:
processors.append(klass())
return bc.ChainedBytecodeProcessor(processors)
def spawn_unittest_runner(self, testables, extra_arguments=None):
"""Spawn test runner process"""
args = [sys.executable, '-m', 'modipyd.tools.unittest_runner']
if extra_arguments:
args.extend(extra_arguments)
if self.test_runner:
args.extend(['-r', self.test_runner])
for t in testables:
args.append(t.filename)
args = [str(arg) for arg in args]
if sys.platform == "win32":
# Avoid argument parsing problem in
# windows, DOS platform
args = ['"%s"' % arg for arg in args]
# Manipulate PYTHONPATH environment variable so that
# the unittest runner can find an appropriate modipyd package.
environ = os.environ.copy()
path = os.path.join(os.path.dirname(modipyd.__file__), '..')
try:
environ['PYTHONPATH'] += (':' + path)
except KeyError:
environ['PYTHONPATH'] = path
LOGGER.debug(
"Spawn test runner process: PYTHONPATH=%s %s" %
(path, ' '.join(args)))
return os.spawnve(os.P_WAIT, sys.executable, args, environ)
def run():
parser = make_option_parser()
options, args = parser.parse_args()
try:
make_application(options, args or '.').run()
except KeyboardInterrupt:
LOGGER.debug('Keyboard Interrupt', exc_info=True)
def collect_module_code(filepath_or_list, search_path=None):
resolver = ModuleNameResolver(search_path)
for filename, typebits in collect_python_module_file(filepath_or_list):
try:
yield read_module_code(filename,
search_path=search_path, typebits=typebits,
resolver=resolver, allow_compilation_failure=True)
except ImportError:
LOGGER.debug("Couldn't import file", exc_info=True)
def run():
"""Standalone program interface"""
parser = make_option_parser()
(options, args) = parser.parse_args()
application = make_application(options, args or '.')
try:
application.run()
except KeyboardInterrupt:
LOGGER.debug('Keyboard Interrupt', exc_info=True)
def descriptors(self):
"""
All the monitoring modules. This is dictionary,
maps name and module descriptors.
"""
if self.__descriptors is None:
self.__descriptors = {}
entries = list(self.refresh())
LOGGER.debug("%d descriptoes" % len(entries))
return self.__descriptors
def collect_module_code(filepath_or_list, search_path=None):
resolver = ModuleNameResolver(search_path)
for filename, typebits in collect_python_module_file(filepath_or_list):
# Since changing .py file is not reflected by .pyc, .pyo quickly,
# the plain .py file takes first prioriry.
try:
yield read_module_code(filename,
search_path=search_path, typebits=typebits,
resolver=resolver, allow_compilation_failure=True)
except ImportError:
LOGGER.debug("Couldn't import file", exc_info=True)
def invoke_plugins(self, event, monitor):
context = dict(self.variables)
for plugin in self.plugins:
try:
ret = plugin(event, monitor, context)
# the plugin object can return (but not required)
# a callable object. It is called with no arguments
if callable(ret):
ret()
except StandardError:
LOGGER.warn(
"Exception occurred while invoking plugin",
exc_info=True)
def reload(self, descriptors, co=None):
"""
Reload module code, update dependency graph
"""
LOGGER.info("Reload module descriptor '%s' at %s" % (self.name, relativepath(self.filename)))
try:
self.module_code.reload(co)
except SyntaxError:
# SyntaxError is OK
LOGGER.warn("SyntaxError found in %s" % self.filename, exc_info=True)
else:
self.update_dependencies(descriptors)
def refresh(self):
assert isinstance(self.paths, (tuple, list))
assert isinstance(self.__descriptors, dict)
assert isinstance(self.__filenames, dict)
assert isinstance(self.__failures, set)
# localize variable access to minimize overhead
# and to reduce the visual noise.
descriptors = self.__descriptors
filenames = self.__filenames
failures = self.__failures
# ``monitor()`` updates all entries and
# removes deleted entries.
for modified in self.monitor():
yield modified
# For now, only need to check new entries.
resolver = ModuleNameResolver(self.search_path)
newcomers = []
for filename, typebits in collect_python_module_file(self.paths):
if filename in filenames or filename in failures:
continue
try:
mc = read_module_code(filename, typebits=typebits,
search_path=self.search_path,
resolver=resolver,
allow_compilation_failure=True,
allow_standalone=True)
except ImportError:
LOGGER.debug("Couldn't import file", exc_info=True)
failures.add(filename)
continue
else:
desc = ModuleDescriptor(mc)
self.add(desc)
# modifieds += new entries
newcomers.append(desc)
LOGGER.debug("Added: %s" % desc.describe())
if newcomers:
# Since there are some entries already refer new entry,
# we need to update dependencies of all entries
for desc in descriptors.itervalues():
desc.update_dependencies(descriptors)
# Notify caller what entries are appended
for desc in newcomers:
yield Event(Event.MODULE_CREATED, desc)
def read_module_code(filename, typebits=None, search_path=None,
resolver=None,
allow_compilation_failure=False,
allow_standalone=False):
"""
Read python module file, and return ``ModuleCode`` instance.
If *typebits* argument is not ``None``, *filename* must be
filepath without file extention.
If *typebits* argument is ``None``, it is detected by filename.
"""
if typebits is None:
filename, _, typebits = module_file_typebits(filename)
if resolver is None:
resolver = ModuleNameResolver(search_path)
code = None
try:
# Since editing .py files will not affect .pyc and .pyo files soon,
# give priority to .py files.
if typebits & PYTHON_SOURCE_MASK:
# .py
sourcepath = filename + '.py'
code = compile_source(sourcepath)
elif typebits & (PYTHON_OPTIMIZED_MASK | PYTHON_COMPILED_MASK):
# .pyc, .pyo
if typebits & PYTHON_OPTIMIZED_MASK:
sourcepath = filename + '.pyo'
else:
sourcepath = filename + '.pyc'
code = load_compiled(sourcepath)
else:
assert False, "illegal typebits: %d" % typebits
except (SyntaxError, ImportError):
LOGGER.warn(
"Exception occurred while loading compiled bytecode",
exc_info=True)
if not allow_compilation_failure:
raise
try:
module_name, package_name = resolver.resolve(sourcepath)
except ImportError:
if not allow_standalone:
raise
module_name = filepath_to_identifier(sourcepath)
package_name = None
return ModuleCode(module_name, package_name, sourcepath, code)
def remove(self, descriptor):
"""Remove *descriptor*, and clear dependencies"""
descriptors, filenames = self.descriptors, self.__filenames
filename = splitext(descriptor.filename)[0]
if (descriptor.name not in descriptors or
filename not in filenames):
raise KeyError(
"No monitoring descriptor '%s'" % \
descriptor.name)
LOGGER.debug("Removed: %s" % descriptor.describe())
descriptor.clear_dependencies()
del descriptors[descriptor.name]
del filenames[filename]
def test_autotest_plugin(self):
monitor = Monitor(__file__)
descriptor = ModuleDescriptor(read_module_code(__file__))
event = Event(Event.MODULE_MODIFIED, descriptor)
plugin = FakeAutotest(event, monitor, {});
self.assertEqual(descriptor, plugin.descriptor)
self.assertTrue(callable(plugin))
plugin()
self.assertEqual([descriptor], plugin.testables)
self.assertEqual(['--loglevel', LOGGER.getEffectiveLevel()], plugin.extra_arguments)
def start(self, interval=1.0, refresh_factor=5):
if refresh_factor < 1:
raise RuntimeError("refresh_factor must be greater or eqaul to 1")
if interval <= 0:
raise RuntimeError("interval must not be negative or 0")
descriptors = self.descriptors
if LOGGER.isEnabledFor(logging.INFO):
desc = "\n".join([
desc.describe(indent=4)
for desc in descriptors.itervalues()])
LOGGER.info("Monitoring:\n%s" % desc)
# Prior to Python 2.5, the ``yield`` statement is not
# allowed in the ``try`` clause of a ``try ... finally``
# construct.
try:
self.monitoring = True
times = 0
while descriptors and self.monitoring:
time.sleep(interval)
times += 1
if times % refresh_factor == 0:
monitor = self.refresh()
else:
monitor = self.monitor()
for modified in monitor:
if not self.monitoring:
break
yield modified
else:
LOGGER.info("Terminating monitor %s" % str(self))
except:
self.monitoring = False
raise
def install_plugin(self, plugin):
"""
Install a plugin specified by *plugin*. The *plugin* argument
must be callable object (e.g. function, class) or a qualified
name of the plugin itself.
Read the ``modipyd.application.plugins`` module documentation
for the plugin architecture details.
"""
if isinstance(plugin, basestring):
try:
plugin = import_component(plugin)
except (ImportError, AttributeError):
LOGGER.error("Loading plugin '%s' failed" % plugin)
raise
if not callable(plugin):
raise TypeError("The plugin must be callable object")
if hasattr(plugin, 'func_code'):
LOGGER.info("Loading plugin: %s" % plugin.func_code.co_name)
else:
LOGGER.info("Loading plugin: %s" % plugin)
self.plugins.append(plugin)
def collect_unittest(paths):
suite = unittest.TestSuite()
loader = unittest.defaultTestLoader
resolver = resolve.ModuleNameResolver()
paths = utils.sequence(paths)
for filepath in paths:
try:
name, package = resolver.resolve(filepath)
except ImportError:
# .py file not in the search path
name = filepath_to_identifier(filepath)
package = None
try:
if package:
module = utils.import_module(name)
else:
module = imp.load_source(name, filepath)
except ImportError:
LOGGER.warn("ImportError occurred while loading module", exc_info=True)
else:
tests = loader.loadTestsFromModule(module)
if tests.countTestCases():
suite.addTest(tests)
LOGGER.info("Found %d test(s) in module '%s'" % (tests.countTestCases(), module.__name__))
else:
LOGGER.warn("No tests found in module '%s'" % module.__name__)
return suite
def make_application(options, filepath):
# options handling
if options.verbosity > 0:
LOGGER.setLevel(logging.INFO)
if options.verbosity > 1:
LOGGER.setLevel(logging.DEBUG)
# So many projects contain its modules and packages at
# the top level directory, modipyd inserts current directory
# in ``sys.path`` module search path variable for convenience.
sys.path.insert(0, os.getcwd())
# Create Application instance, Install plugins
application = Application(filepath)
for plugin in options.plugins:
application.install_plugin(plugin)
# Predefine variables
variables = {}
for var in options.defines:
i = var.find('=')
if i == -1:
variables[var] = ''
else:
variables[var[:i]] = var[i+1:]
if variables:
import pprint
application.update_variables(variables)
LOGGER.info(
"Predefined variables: %s" % pprint.pformat(variables))
# Load configuration (startup) file
for rcfile in find_startup_files(os.environ, options.rcfile):
LOGGER.info("Loading startup file from %s" % rcfile)
execfile(rcfile, globals(), {'application': application})
return application
def __call__(self):
# Walking dependency graph in imported module to
# module imports order.
testables = []
for desc in self.descriptor.walk_dependency_graph(reverse=True):
LOGGER.info("-> Affected: %s" % desc.name)
if has_subclass(desc, unittest.TestCase):
LOGGER.debug("-> unittest.TestCase detected: %s" % desc.name)
testables.append(desc)
# Runntine tests
if testables:
# We can reload affected modules manually and run
# all TestCase in same process. Running another process,
# however, is simple and perfect solution.
if LOGGER.isEnabledFor(logging.INFO):
desc = ', '.join([x.name for x in testables])
LOGGER.info("Running UnitTests: %s" % desc)
# Propagates the level of modipyd.LOGGER to
# the unittest runner subprocess.
extra = ['--loglevel', LOGGER.getEffectiveLevel()]
self.spawn_unittest_runner(testables, extra)
def has_subclass(module_descriptor, baseclass):
"""
Return ``True`` if the module has a class
derived from *baseclass*
"""
# We can't use ``unittest.TestLoader`` to loading tests,
# bacause ``TestLoader`` imports (execute) module code.
# If imported/executed module have a statement such as
# ``sys.exit()``, ...program exit!
if not isinstance(baseclass, (type, types.ClassType)):
raise TypeError(
"The baseclass argument must be instance of type or class, "
"but was instance of %s" % type(baseclass))
modcode = module_descriptor.module_code
assert modcode
# How to check unittest.TestCase
# ============================================
# 1. For all class definition in module code
# 2. Check class is derived from base class(s)
# 3. Check base class(s) is imported from another module
# 4. Load base class(s) from that module
# Notes: Assume the module contains base class does not have
# a dangerous code such as ``sys.exit``.
# 5. Check loaded class is *baseclass* or its subclass
# Construct imported symbols.
# This is used in phase 3.
symbols = dict([(imp[0], imp) for imp in modcode.context['imports']])
# 1. For all class definition in module code
for klass in modcode.context['classdefs']:
# 2. Check class is derived from base class(s)
bases = klass[1]
if not bases:
continue
# 3. Check base class(s) is imported from another module
for base in bases:
# Search imported symbol that is class name or module name
if '.' in base:
names = list(split_module_name(base))
else:
names = [base]
import_ = symbols.get(names[0])
if import_ is None:
# Not an imported base class
continue
# Convert a name to a qualified module name
#
# 1. Resolve import alias if exists
# 2. Qualify name as full module name
# 3. Resolve relative module name
#
level = import_[2]
names[0] = import_[1]
fqn = '.'.join(names)
fqn = resolve_relative_modulename(fqn, modcode.package_name, level)
assert '.' in fqn, "fqn must be a qualified module fqn"
LOGGER.debug("'%s' is derived from '%s'" % (module_descriptor.name, fqn))
try:
try:
klass = utils.import_component(fqn)
except ImportError:
if level == -1 and modcode.package_name:
# The qualified name may be relative to current package.
fqn = '.'.join((modcode.package_name, fqn))
klass = utils.import_component(fqn)
else:
raise
except (ImportError, AttributeError):
LOGGER.warn("Exception occurred "
"while importing component '%s'" % fqn,
exc_info=True)
else:
# 5. Check loaded class is specified class or its subclass
if isinstance(klass, (type, types.ClassType)) and \
issubclass(klass, baseclass):
return True
return False
def update_dependencies(self, descriptors):
LOGGER.debug("Update dependencies of '%s'" % self.name)
_update_module_dependencies(self, descriptors)
def has_subclass(module_descriptor, baseclass):
"""
Return ``True`` if the module has a class
derived from *baseclass*
"""
# We can't use ``unittest.TestLoader`` to loading tests,
# bacause ``TestLoader`` imports (execute) module code.
# If imported/executed module have a statement such as
# ``sys.exit()``, ...program exit!
if not isinstance(baseclass, (type, types.ClassType)):
raise TypeError(
"The baseclass argument must be instance of type or class, "
"but was instance of %s" % type(baseclass))
modcode = module_descriptor.module_code
assert modcode
# How to check unittest.TestCase
# ============================================
# 1. For all class definition in module code
# 2. Check class is derived from base class(s)
# 3. Check base class(s) is imported from another module
# 4. Load base class(s) from that module
# Notes: Assume the module contains base class does not have
# a dangerous code such as ``sys.exit``.
# 5. Check loaded class is *baseclass* or its subclass
# Construct imported symbols.
# This is used in phase 3.
symbols = dict([(imp[0], imp) for imp in modcode.context['imports']])
# 1. For all class definition in module code
for klass in modcode.context['classdefs']:
# 2. Check class is derived from base class(s)
bases = klass[1]
if not bases:
continue
# 3. Check base class(s) is imported from another module
for base in bases:
# Search imported symbol that is class name or module name
symbol = base
if '.' in symbol:
symbol = split_module_name(symbol)[0]
import_ = symbols.get(symbol)
if import_ is None:
continue
# Convert name to a qualified module name
name, level = base, import_[2]
parent = split_module_name(import_[1])[0]
if parent:
name = '.'.join((parent, name))
name = resolve_relative_modulename(
name, modcode.package_name, level)
assert '.' in name, "name must be a qualified module name"
LOGGER.debug("'%s' is derived from '%s'" % (base, name))
try:
klass = utils.import_component(name)
except ImportError:
klass = None
exc = sys.exc_info()[:]
if level == -1 and modcode.package_name:
# Try to resolve a name as relative module name.
try:
name2 = '.'.join((modcode.package_name, name))
klass = utils.import_component(name2)
except:
LOGGER.warn(
"Exception occurred while importing module '%s'" % name2,
exc_info=True)
if not klass:
LOGGER.warn(
"Exception occurred while importing module '%s'" % name,
exc_info=exc)
# Make sure to delete the traceback to avoid creating cycles.
del exc
except AttributeError:
LOGGER.warn(
"Exception occurred while importing module '%s'" % name,
exc_info=True)
else:
# 5. Check loaded class is specified class or its subclass
if isinstance(klass, (type, types.ClassType)) and \
issubclass(klass, baseclass):
return True
return False
def run(self):
monitor = Monitor(self.paths)
for event in monitor.start():
LOGGER.info("%s: %s" % (TYPE_STRINGS[event.type],
event.descriptor.describe(indent=4)))
self.invoke_plugins(event, monitor)
if __name__ == "__main__":
parser = OptionParser(usage="usage: %prog [options] file1, file2, ...")
parser.add_option(
"-r",
"--runner",
default="unittest.TextTestRunner",
action="store",
dest="runner",
metavar="CLASS_NAME",
help="qualified name of the unittest.TestRunner subclass " "(default: unittest.TextTestRunner)",
)
parser.add_option(
"--loglevel",
action="store",
type="int",
dest="loglevel",
metavar="LOG_LEVEL",
help="Specifies the lowest-severity log message a logger will handle",
)
options, args = parser.parse_args()
if options.loglevel is not None:
LOGGER.setLevel(options.loglevel)
LOGGER.debug("Execute modipyd.tools.unittest_runner: %s" % " ".join(args))
sys.path.insert(0, os.getcwd())
main(args, options.runner)