def dump_main(ctx):
ctx.stats.timer('main', 'started')
ret = True
if len(ctx.routes.groups()) < 2:
log.error("There is only one group in route list: {0}. "
"sdc recovery could not be made."
.format(ctx.routes.groups()))
return False
try:
ctx.merged_filename = lookup_keys(ctx)
except KeyboardInterrupt:
log.error("Caught Ctrl+C. Terminating.")
ctx.stats.timer('main', 'finished')
return False
log.debug("Merged_filename: %s, address: %s, groups: %s, tmp_dir:%s",
ctx.merged_filename, ctx.address, ctx.groups, ctx.tmp_dir)
if ctx.dry_run:
return ret
if ctx.custom_recover == '':
recover(ctx)
else:
import imp
log.debug("Loading module: {0}".format(ctx.custom_recover))
imp.acquire_lock()
custom_recover = imp.load_source('custom_recover', ctx.custom_recover)
imp.release_lock()
custom_recover.recover(ctx)
ctx.stats.timer('main', 'finished')
return ret
def setImage(width, height, image_data):
global _imageData
imp.acquire_lock()
_imageWidth = width
_imageHeight = height
_imageData = bytearray(image_data)
imp.release_lock()
def testLock(self):
LOOPS = 50
# The import lock may already be held, e.g. if the test suite is run
# via "import test.autotest".
lock_held_at_start = imp.lock_held()
self.verify_lock_state(lock_held_at_start)
for i in range(LOOPS):
imp.acquire_lock()
self.verify_lock_state(True)
for i in range(LOOPS):
imp.release_lock()
# The original state should be restored now.
self.verify_lock_state(lock_held_at_start)
if not lock_held_at_start:
try:
imp.release_lock()
except RuntimeError:
pass
else:
self.fail("release_lock() without lock should raise "
"RuntimeError")
def view_import(name, globals={}, locals={}, fromlist=[], level=0):
"""the drop-in replacement for __import__, that optionally imports
locally as well.
"""
# don't override nested imports
save_import = builtin_mod.__import__
builtin_mod.__import__ = local_import
if imp.lock_held():
# this is a side-effect import, don't do it remotely, or even
# ignore the local effects
return local_import(name, globals, locals, fromlist, level)
imp.acquire_lock()
if local:
mod = local_import(name, globals, locals, fromlist, level)
else:
raise NotImplementedError("remote-only imports not yet implemented")
imp.release_lock()
key = name+':'+','.join(fromlist or [])
if level <= 0 and key not in modules:
modules.add(key)
if not quiet:
if fromlist:
print("importing %s from %s on engine(s)"%(','.join(fromlist), name))
else:
print("importing %s on engine(s)"%name)
results.append(self.apply_async(remote_import, name, fromlist, level))
# restore override
builtin_mod.__import__ = save_import
return mod
def load_plugin(plugin_file, valid_base_class):
"""
Load a plugin into memory and extract its single interface class.
"""
# construct a uuid and use this as the module name to ensure
# that each import is unique
import uuid
module_uid = uuid.uuid4().hex
module = None
try:
imp.acquire_lock()
module = imp.load_source(module_uid, plugin_file)
except Exception:
# dump out the callstack for this one -- to help people get good messages when there is a plugin error
(exc_type, exc_value, exc_traceback) = sys.exc_info()
message = ""
message += "Failed to load plugin %s. The following error was reported:\n" % plugin_file
message += "Exception: %s - %s\n" % (exc_type, exc_value)
message += "Traceback (most recent call last):\n"
message += "\n".join(traceback.format_tb(exc_traceback))
raise TankError(message)
finally:
imp.release_lock()
# cool, now validate the module
found_classes = list()
introspection_error_reported = None
try:
for var in dir(module):
value = getattr(module, var)
if isinstance(value, type) and issubclass(
value, valid_base_class) and value != valid_base_class:
found_classes.append(value)
except Exception, e:
introspection_error_reported = str(e)
def importer():
imp.acquire_lock()
sys.modules[fake_module_name] = partial_module
import_started.set()
time.sleep(0.01) # Give the other thread time to try and acquire.
sys.modules[fake_module_name] = complete_module
imp.release_lock()
def other_thread():
imp.acquire_lock() # 3
assert imp.lock_held()
lock_held.release() # 4
test_complete.acquire() # 7
imp.release_lock() # 8
lock_released.release() # 9
def _load_plugin(self, plugin_id, path=None):
manifest = self.manifests[plugin_id]
path = path or manifest.src_path or self.plugin_path(plugin_id)
imp.acquire_lock()
try:
finder = PluginModuleFinder(path)
sys.meta_path.append(finder)
main_module = manifest.main_module or plugin_id
result = importlib.import_module(main_module)
plugin_exports = [main_module]
plugin_modules = []
if manifest.exported_modules:
plugin_exports.extend(manifest.exported_modules)
for module_name in finder.plugin_modules:
if hasattr(sys.modules[module_name], '__path__'):
pkg_prefix = module_name + '.'
should_remove = not any(
name.startswith(pkg_prefix) for name in plugin_exports)
else:
should_remove = module_name not in plugin_exports
if should_remove:
sys.modules.pop(module_name, None)
else:
plugin_modules.append(module_name)
self.module_refcount[module_name] += 1
self.plugin_modules[plugin_id] = plugin_modules
return result
finally:
sys.meta_path.remove(finder)
imp.release_lock()
def _import_process_module(filepath, fullname):
"""
Returns the specified process module.
Arguments:
filepath (str)
- The file path to process module.
fullname (str)
- The fully qualified name of the process module:
i.e., {processing}.processes.modules.{process_name}
Returns:
(module) -- The process module.
"""
mod_dir, mod_name = _os.path.split(filepath)
fh, path, desc = _imp.find_module(mod_name, [mod_dir])
try:
_imp.acquire_lock()
try:
proc_mod = _imp.load_module(fullname, fh, path, desc)
finally:
_imp.release_lock()
finally:
if fh:
fh.close()
return proc_mod
def do_import(self, name, globals=None, locals=None, fromlist=None, level=None):
# Holding the import lock around the whole import process matches what
# Python does internally. This does mean that the machinery of loading a slow
# import blocks the import of an already loaded module in a different thread.
# You could imagine trying to do the lookup without the lock and locking only
# for loading, but ensuring the safety of that would be quite complex
imp.acquire_lock()
try:
names = name.split('.')
module, local = self.__import_recurse(names)
if fromlist != None:
# In 'from a.b import c', if a.b.c doesn't exist after loading a.b, The built-in
# __import__ will try to load a.b.c as a module; do the same here.
for fromname in fromlist:
if fromname == "*":
try:
all = getattr(module, "__all__")
for allname in all:
self.__ensure_from_list_item(name, allname, module, local)
except AttributeError:
pass
else:
self.__ensure_from_list_item(name, fromname, module, local)
return module
elif local:
return self.__modules[names[0]]
else:
return sys.modules[names[0]]
finally:
imp.release_lock()
def __import__(*args, **kwargs):
"""
__import__(name, globals=None, locals=None, fromlist=(), level=0) -> object
Normally python protects imports against concurrency by doing some locking
at the C level (at least, it does that in CPython). This function just
wraps the normal __import__ functionality in a recursive lock, ensuring that
we're protected against greenlet import concurrency as well.
"""
if len(args) > 0 and not issubclass(type(args[0]),
_allowed_module_name_types):
# if a builtin has been acquired as a bound instance method,
# python knows not to pass 'self' when the method is called.
# No such protection exists for monkey-patched builtins,
# however, so this is necessary.
args = args[1:]
if not __lock_imports:
return _import(*args, **kwargs)
module_lock = __module_lock(args[0]) # Get a lock for the module name
imp.acquire_lock()
try:
module_lock.acquire()
try:
result = _import(*args, **kwargs)
finally:
module_lock.release()
finally:
imp.release_lock()
return result
def load_module(self, module_name):
thread_count = threading.active_count()
if thread_count != 1 :
while imp.lock_held():
pass
imp.acquire_lock()
module = sys.modules.setdefault(module_name, imp.new_module(module_name))
if thread_count != 1:
imp.release_lock() # not sure when to release; the following doesn't use shared resources (e.g. sys.modules) other than the module itself
source, filepath = self.source.pop(module_name)
module_code = compile(source, module_name, "exec")
is_package = True if len(module_name.split('.')) > 1 else False # not sure, but seems accurate
module.__file__ = filepath
module.__loader__ = self
if is_package:
module.__path__ = []
module.__package__ = module_name
else:
module.__package__ = module_name.split('.', 1)[0]
exec module_code in module.__dict__
#imp.release_lock() # it might be more correct to release the lock here instead of above.
return module
def load_from_cache(self, name):
module_data = self.module_cache[name]
if module_data.code is None:
return self.load_as_normal(name)
if sys.version_info[0] == 2:
# Load, ignoring checksum and time stamp.
# 8 is header length (12 in py3, if we need that someday)
code = marshal.loads(module_data.code[8:])
else:
code = module_data.code
imp.acquire_lock() # Required in threaded applications
print('{} load {}'.format(world.rank, name))
module = imp.new_module(name)
# To properly handle circular and submodule imports, we must
# add the module before executing its code:
sys.modules[name] = module
# Set data like __path__, __file__ etc. which are defined
# by the loader and not the code itself:
for var in module_data.vars:
setattr(module, var, module_data.vars[var])
exec(code, module.__dict__)
imp.release_lock()
return module
def get_image_data():
img_data_copy = bytearray()
if _imageData:
imp.acquire_lock()
img_data_copy = bytearray(_imageData)
imp.release_lock()
return (_imageWidth, _imageHeight, img_data_copy)
def __import__(*args, **kwargs):
"""
Normally python protects imports against concurrency by doing some locking
at the C level (at least, it does that in CPython). This function just
wraps the normal __import__ functionality in a recursive lock, ensuring that
we're protected against greenlet import concurrency as well.
"""
if len(args) > 0 and not issubclass(type(args[0]), _allowed_module_name_types):
# if a builtin has been acquired as a bound instance method,
# python knows not to pass 'self' when the method is called.
# No such protection exists for monkey-patched builtins,
# however, so this is necessary.
args = args[1:]
# TODO: It would be nice not to have to acquire the locks
# if the module is already imported (in sys.modules), but the interpretation
# of the arguments is somewhat complex
imp.acquire_lock()
try:
_g_import_lock.acquire()
try:
result = _import(*args, **kwargs)
finally:
_g_import_lock.release()
finally:
imp.release_lock()
return result
def find_module(self, fullname, path=None):
"""
PEP-302 finder.find_module() method for the ``sys.meta_path`` hook.
fullname fully qualified name of the module
path None for a top-level module, or package.__path__ for submodules or subpackages.
Return a loader object if the module was found, or None if it wasn't. If find_module() raises
an exception, it will be propagated to the caller, aborting the import.
"""
# Acquire the interpreter's import lock for the current thread. Tis
# lock should be used by import hooks to ensure thread-safety when
# importing modules.
imp.acquire_lock()
# TODO rewrite this method.
module_loader = None # None means - no module found in this importer.
try:
print fullname
if fullname in self._pyz_archive.toc:
print '... found'
# Tell the import machinery to use self.load_module() to load the module.
module_loader = self
else:
print '... found'
finally:
# Release the interpreter's import lock.
imp.release_lock()
return module_loader
def _get_loader(mod_name, path=None):
"""Retrieve a PEP 302 loader for the given module or package
If the module or package is accessible via the normal import
mechanism, a wrapper around the relevant part of that machinery
is returned.
Non PEP 302 mechanisms (e.g. the Windows registry) used by the
standard import machinery to find files in alternative locations
are partially supported, but are searched AFTER sys.path. Normally,
these locations are searched BEFORE sys.path, preventing sys.path
entries from shadowing them.
For this to cause a visible difference in behaviour, there must
be a module or package name that is accessible via both sys.path
and one of the non PEP 302 file system mechanisms. In this case,
the emulation will find the former version, while the builtin
import mechanism will find the latter.
Items of the following types can be affected by this discrepancy:
imp.C_EXTENSION
imp.PY_SOURCE
imp.PY_COMPILED
imp.PKG_DIRECTORY
"""
try:
loader = sys.modules[mod_name].__loader__
except (KeyError, AttributeError):
loader = None
if loader is None:
imp.acquire_lock()
try:
# Module not in sys.modules, or uses an unhooked loader
parts = mod_name.rsplit(".", 1)
if len(parts) == 2:
# Sub package, so use parent package's path
pkg_name, sub_name = parts
if pkg_name and pkg_name[0] != ".":
if path is not None:
raise ImportError("Path argument must be None " "for a dotted module name")
pkg = _get_package(pkg_name)
try:
path = pkg.__path__
except AttributeError:
raise ImportError(pkg_name + " is not a package")
else:
raise ImportError("Relative import syntax is not " "supported by _get_loader()")
else:
# Top level module, so stick with default path
sub_name = mod_name
for importer in sys.meta_path:
loader = importer.find_module(mod_name, path)
if loader is not None:
# Found a metahook to handle the module
break
else:
# Handling via the standard path mechanism
loader = _get_path_loader(mod_name, path)
finally:
imp.release_lock()
return loader
def dump_main(ctx):
ctx.stats.timer('main', 'started')
ret = True
if len(ctx.routes.groups()) < 2:
log.error("There is only one group in route list: {0}. "
"sdc recovery could not be made.".format(
ctx.routes.groups()))
return False
try:
ctx.merged_filename = lookup_keys(ctx)
except KeyboardInterrupt:
log.error("Caught Ctrl+C. Terminating.")
ctx.stats.timer('main', 'finished')
return False
log.debug("Merged_filename: %s, address: %s, groups: %s, tmp_dir:%s",
ctx.merged_filename, ctx.address, ctx.groups, ctx.tmp_dir)
if ctx.dry_run:
return ret
if ctx.custom_recover == '':
recover(ctx)
else:
import imp
log.debug("Loading module: {0}".format(ctx.custom_recover))
imp.acquire_lock()
custom_recover = imp.load_source('custom_recover', ctx.custom_recover)
imp.release_lock()
custom_recover.recover(ctx)
ctx.stats.timer('main', 'finished')
return ret
def __init__(self, to=None, defaultPk=None, **kwargs):
super(ForeignObjectValidatorMixin, self).__init__(
to=to, defaultPk=defaultPk, **kwargs)
self._defaultPk = defaultPk
if to is None:
return
if 'instance' in kwargs:
# Replace _to method to foreign constructor
if isinstance(to, string_types):
import sys
to_path = to.split('.')
object_rel = to_path.pop()
package_rel = '.'.join(to_path)
if PY2:
import imp as _imp
else:
import _imp
_imp.acquire_lock()
module1 = __import__('.'.join(to_path))
_imp.release_lock()
try:
self._to = getattr(sys.modules[package_rel], object_rel)
except AttributeError:
raise AttributeError('Package "%s" not contain model %s' %
(package_rel, object_rel))
else:
self._to = to
def tangelo_import(*args, **kwargs):
"""
When we are asked to import a module, if we get an import error and the
calling script is one we are serving (not one in the python libraries), try
again in the same directory as the script that is calling import.
It seems like we should use sys.meta_path and combine our path with the
path sent to imp.find_module. This requires duplicating a bunch of logic
from the imp module and is actually heavier than this technique.
:params: see __builtin__.__import__
"""
try:
return builtin_import(*args, **kwargs)
except ImportError:
if not hasattr(cherrypy.thread_data, "modulepath"):
raise
path = os.path.abspath(cherrypy.thread_data.modulepath)
root = os.path.abspath(cherrypy.config.get("webroot"))
result = None
imp.acquire_lock()
oldpath = sys.path
try:
# If the module's path isn't in the system path but is in our
# serving area, temporarily add it and try the import again.
if path not in sys.path and (path == root or
path.startswith(root + os.path.sep)):
sys.path = [path] + sys.path
result = builtin_import(*args, **kwargs)
finally:
sys.path = oldpath
imp.release_lock()
if result is not None:
return result
raise
def load_plugins():
"""
Load all found plugins
:return:
"""
found, disabled = find_plugins()
imp.acquire_lock()
for plugname in disabled:
sys.stdout.write("plugin {} disabled\n".format(plugname))
for plugname in found:
try:
sys.stdout.write("loading plugin {}\n".format(plugname))
with open(found[plugname], "rb") as plugfile:
plugmod = imp.load_module(
plugname, plugfile,
found[plugname].encode(sys.getfilesystemencoding()),
(".py", "r", imp.PY_SOURCE))
if hasattr(plugmod, "plugin_start"):
newname = plugmod.plugin_start()
PLUGINS[newname and unicode(newname) or plugname] = plugmod
except Exception as plugerr:
if __debug__:
print_exc()
else:
sys.stderr.write(
'%s: %s\n' % (plugname, plugerr)
) # appears in %TMP%/EDMarketConnector.log in packaged Windows app
imp.release_lock()
def _run_module_code(code, init_globals=None,
mod_name=None, mod_fname=None,
mod_loader=None, alter_sys=False):
"""Helper for run_module"""
# Set up the top level namespace dictionary
if alter_sys:
# Modify sys.argv[0] and sys.module[mod_name]
temp_module = imp.new_module(mod_name)
mod_globals = temp_module.__dict__
saved_argv0 = sys.argv[0]
restore_module = mod_name in sys.modules
if restore_module:
saved_module = sys.modules[mod_name]
imp.acquire_lock()
try:
sys.argv[0] = mod_fname
sys.modules[mod_name] = temp_module
try:
_run_code(code, mod_globals, init_globals,
mod_name, mod_fname, mod_loader)
finally:
sys.argv[0] = saved_argv0
if restore_module:
sys.modules[mod_name] = saved_module
else:
del sys.modules[mod_name]
finally:
imp.release_lock()
# Copy the globals of the temporary module, as they
# may be cleared when the temporary module goes away
return mod_globals.copy()
else:
# Leave the sys module alone
return _run_code(code, {}, init_globals,
mod_name, mod_fname, mod_loader)
def oss_to_epoch(timestamp):
# given a time string like: 2014-05-15T11:18:32.000Z
# convert it to an epoch time.
imp.acquire_lock()
ts = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.000Z").timetuple()
imp.release_lock()
return (int)(time.mktime(ts))
def _populate(self):
"""
Fill in all the cache information. This method is threadsafe, in the
sense that every caller will see the same state upon return, and if the
cache is already initialised, it does no work.
"""
if self.loaded:
return
# Note that we want to use the import lock here - the app loading is
# in many cases initiated implicitly by importing, and thus it is
# possible to end up in deadlock when one thread initiates loading
# without holding the importer lock and another thread then tries to
# import something which also launches the app loading. For details of
# this situation see #18251.
imp.acquire_lock()
try:
if self.loaded:
return
for app_name in settings.INSTALLED_APPS:
if app_name in self.handled:
continue
self.load_app(app_name, True)
if not self.nesting_level:
for app_name in self.postponed:
self.load_app(app_name)
self.loaded = True
finally:
imp.release_lock()
def get_app(self, app_label, emptyOK=False):
"""
Returns the module containing the models for the given app_label.
Returns None if the app has no models in it and emptyOK is True.
Raises UnavailableApp when set_available_apps() in in effect and
doesn't include app_label.
"""
self._populate()
imp.acquire_lock()
try:
for app_name in settings.INSTALLED_APPS:
if app_label == app_name.split('.')[-1]:
mod = self.load_app(app_name, False)
if mod is None and not emptyOK:
raise ImproperlyConfigured(
"App with label %s is missing a models.py module."
% app_label)
if self.available_apps is not None and app_label not in self.available_apps:
raise UnavailableApp(
"App with label %s isn't available." % app_label)
return mod
raise ImproperlyConfigured("App with label %s could not be found" %
app_label)
finally:
imp.release_lock()
def testLock(self):
LOOPS = 50
# The import lock may already be held, e.g. if the test suite is run
# via "import test.autotest".
lock_held_at_start = imp.lock_held()
self.verify_lock_state(lock_held_at_start)
for i in range(LOOPS):
imp.acquire_lock()
self.verify_lock_state(True)
for i in range(LOOPS):
imp.release_lock()
# The original state should be restored now.
self.verify_lock_state(lock_held_at_start)
if not lock_held_at_start:
try:
imp.release_lock()
except RuntimeError:
pass
else:
self.fail("release_lock() without lock should raise "
"RuntimeError")
def load_plugin(plugin_file, valid_base_class):
"""
Load a plugin into memory and extract its single interface class.
"""
# construct a uuid and use this as the module name to ensure
# that each import is unique
import uuid
module_uid = uuid.uuid4().hex
module = None
try:
imp.acquire_lock()
module = imp.load_source(module_uid, plugin_file)
except Exception:
# dump out the callstack for this one -- to help people get good messages when there is a plugin error
(exc_type, exc_value, exc_traceback) = sys.exc_info()
message = ""
message += "Failed to load plugin %s. The following error was reported:\n" % plugin_file
message += "Exception: %s - %s\n" % (exc_type, exc_value)
message += "Traceback (most recent call last):\n"
message += "\n".join( traceback.format_tb(exc_traceback))
raise TankError(message)
finally:
imp.release_lock()
# cool, now validate the module
found_classes = list()
introspection_error_reported = None
try:
for var in dir(module):
value = getattr(module, var)
if isinstance(value, type) and issubclass(value, valid_base_class) and value != valid_base_class:
found_classes.append(value)
except Exception, e:
introspection_error_reported = str(e)
def __import__(*args, **kwargs):
"""
Normally python protects imports against concurrency by doing some locking
at the C level (at least, it does that in CPython). This function just
wraps the normal __import__ functionality in a recursive lock, ensuring that
we're protected against greenlet import concurrency as well.
"""
if len(args) > 0 and not issubclass(type(args[0]), _allowed_module_name_types):
# if a builtin has been acquired as a bound instance method,
# python knows not to pass 'self' when the method is called.
# No such protection exists for monkey-patched builtins,
# however, so this is necessary.
args = args[1:]
# TODO: It would be nice not to have to acquire the locks
# if the module is already imported (in sys.modules), but the interpretation
# of the arguments is somewhat complex.
if not __lock_imports:
return _import(*args, **kwargs)
imp.acquire_lock()
try:
_g_import_lock.acquire()
try:
result = _import(*args, **kwargs)
finally:
_g_import_lock.release()
finally:
imp.release_lock()
return result
def load_module(self, fullname):
if fullname in sys.modules:
return sys.modules[fullname]
path = self.cache[fullname]
# must insert to sys.modules first, to avoid cyclic imports
try:
imp.acquire_lock()
mod = sys.modules[fullname] = imp.new_module(fullname)
try:
if path is None:
mod.__file__ = "<namespace module>"
mod.__path__ = []
else:
info = imp.find_module(os.path.basename(path), [os.path.dirname(path)])
mod = imp.load_module(fullname, *info)
# replace the stub in sys.modules
sys.modules[fullname] = mod
except Exception:
del sys.modules[fullname]
raise
finally:
imp.release_lock()
return mod
def get_led_data():
led_data_copy = bytearray()
if _ledData:
imp.acquire_lock()
led_data_copy = bytearray(_ledData)
imp.release_lock()
return led_data_copy
def import_pyv8():
# Importing non-existing modules is a bit tricky in Python:
# if we simply call `import PyV8` and module doesn't exists,
# Python will cache this failed import and will always
# throw exception even if this module appear in PYTHONPATH.
# To prevent this, we have to manually test if
# PyV8.py(c) exists in PYTHONPATH before importing PyV8
if 'PyV8' in sys.modules and 'PyV8' not in globals():
# PyV8 was loaded by ST2, create global alias
globals()['PyV8'] = __import__('PyV8')
return
loaded = False
f, pathname, description = imp.find_module('PyV8')
bin_f, bin_pathname, bin_description = imp.find_module('_PyV8')
if f:
try:
imp.acquire_lock()
globals()['_PyV8'] = imp.load_module('_PyV8', bin_f, bin_pathname,
bin_description)
globals()['PyV8'] = imp.load_module('PyV8', f, pathname,
description)
imp.release_lock()
loaded = True
finally:
# Since we may exit via an exception, close fp explicitly.
if f:
f.close()
if bin_f:
bin_f.close()
if not loaded:
raise ImportError('No PyV8 module found')
def loadPlugins(self):
"""Load plugins by iterating files in plugin directories.
"""
plugins = []
for dir in self.directories:
try:
for f in os.listdir(dir):
if f.endswith(".py") and f != "__init__.py":
plugins.append((f[:-3], dir))
except OSError:
print "Failed to access: %s" % dir
continue
fh = None
mod = None
for (name, dir) in plugins:
try:
acquire_lock()
fh, filename, desc = find_module(name, [dir])
old = sys.modules.get(name)
if old is not None:
# make sure we get a fresh copy of anything we are trying
# to load from a new path
del sys.modules[name]
mod = load_module(name, fh, filename, desc)
finally:
if fh:
fh.close()
release_lock()
if hasattr(mod, "__all__"):
attrs = [getattr(mod, x) for x in mod.__all__]
for plug in attrs:
if not issubclass(plug, Plugin):
continue
self._loadPlugin(plug())
def run(self, threadrunlimit=None):
"""
threadrunlimit: only run this many threads at most total,
if None (default) then run all threads
"""
runcount = len(self.threads[self.next_thread:])
if threadrunlimit is not None:
runcount = min(runcount, threadrunlimit)
next_thread = 0
while runcount > 0:
batch = self.threads[next_thread:next_thread + self.maxparallel]
# cannot start threads while imp lock is held.
toLock = imp.lock_held()
if toLock:
imp.release_lock()
# Start all threads in this batch
for thread in batch:
thread.start()
# Wait for them all to finish
for thread in batch:
thread.join
# rest lock state
if toLock:
imp.acquire_lock()
runcount = runcount - len(batch)
next_thread = next_thread + len(batch)
def wait(self, timeout=None):
"""
Return the result, or throw the exception if result is a failure.
It may take an unknown amount of time to return the result, so a
timeout option is provided. If the given number of seconds pass with
no result, a TimeoutError will be thrown.
If a previous call timed out, additional calls to this function will
still wait for a result and return it if available. If a result was
returned or raised on one call, additional calls will return/raise the
same result.
"""
if imp.lock_held():
try:
imp.release_lock()
except RuntimeError:
# The lock is held by some other thread. We should be safe
# to continue.
pass
else:
# If EventualResult.wait() is run during module import, if the
# Twisted code that is being run also imports something the result
# will be a deadlock. Even if that is not an issue it would
# prevent importing in other threads until the call returns.
raise RuntimeError(
"EventualResult.wait() must not be run at module import time.")
try:
result = self._result(timeout)
except TimeoutError:
raise
if isinstance(result, Failure):
result.raiseException()
pass
return result
def load_module(self, fullname, path=None):
imp.acquire_lock()
try:
# PEP302 If there is an existing module object named 'fullname'
# in sys.modules, the loader must use that existing module.
module = sys.modules.get(fullname)
if module is None:
filename = pyi_os_path.os_path_join(sys.prefix,
fullname + self._suffix)
fp = open(filename, 'rb')
module = imp.load_module(fullname, fp, filename,
self._c_ext_tuple)
# Set __file__ attribute.
if hasattr(module, '__setattr__'):
module.__file__ = filename
else:
# Some modules (eg: Python for .NET) have no __setattr__
# and dict entry have to be set.
module.__dict__['__file__'] = filename
except Exception:
# Remove 'fullname' from sys.modules if it was appended there.
if fullname in sys.modules:
sys.modules.pop(fullname)
# Release the interpreter's import lock.
imp.release_lock()
raise # Raise the same exception again.
# Release the interpreter's import lock.
imp.release_lock()
return module
def find_module(self, name, path):
# Ignore anything not in the form kaa.foo, or if kaa.foo is an egg in sys.path.
self.discover_kaa_eggs()
if not name.startswith('kaa.') or name.count('.') > 1 or name in self.kaa_eggs:
return
kaa_base_path = __file__.rsplit('/', 1)[0]
name = name[4:].replace('.', '/')
imp.acquire_lock()
try:
# Scenario 1: kaa.base is an on-disk tree (egg or otherwise)
if not self.zip:
try:
return KaaLoader(imp.find_module(name, [kaa_base_path]))
except ImportError:
pass
# Scenario 2: kaa.base is a zipped egg
if '.egg/' in kaa_base_path and self.zip is not False:
try:
if not self.zip:
self.zip = zipimport.zipimporter(kaa_base_path)
except ImportError:
# Not a valid zipped egg, cache the result so we don't try again.
self.zip = False
else:
if self.zip.find_module('kaa.base.' + name):
# kaa.base.foo found inside egg.
return KaaLoader(self.zip)
finally:
imp.release_lock()
def build(self, source, filename):
imp.acquire_lock()
try:
d = self.get(filename)
if d is not None:
return d
base, ext = os.path.splitext(filename)
name = os.path.join(self.path, base + ".py")
log.debug("writing source to disk (%d bytes)." % len(source))
fd, fn = tempfile.mkstemp(prefix=base,
suffix='.tmp',
dir=self.path)
temp = os.fdopen(fd, 'w')
try:
try:
temp.write("%s\n" % '# -*- coding: utf-8 -*-')
temp.write(source)
finally:
temp.close()
except:
os.remove(fn)
raise
os.rename(fn, name)
log.debug("compiling %s into byte-code..." % filename)
py_compile.compile(name)
return self._load(base, name)
finally:
imp.release_lock()
def load_library(self, flags=None):
# XXX review all usages of 'self' here!
# import it as a new extension module
imp.acquire_lock()
try:
if hasattr(sys, "getdlopenflags"):
previous_flags = sys.getdlopenflags()
try:
if hasattr(sys, "setdlopenflags") and flags is not None:
sys.setdlopenflags(flags)
module = imp.load_dynamic(self.verifier.get_module_name(),
self.verifier.modulefilename)
except ImportError as e:
error = "importing %r: %s" % (self.verifier.modulefilename, e)
raise VerificationError(error)
finally:
if hasattr(sys, "setdlopenflags"):
sys.setdlopenflags(previous_flags)
finally:
imp.release_lock()
#
# call loading_cpy_struct() to get the struct layout inferred by
# the C compiler
self._load(module, 'loading')
#
# the C code will need the <ctype> objects. Collect them in
# order in a list.
revmapping = dict([(value, key)
for (key, value) in self._typesdict.items()])
lst = [revmapping[i] for i in range(len(revmapping))]
lst = list(map(self.ffi._get_cached_btype, lst))
#
# build the FFILibrary class and instance and call _cffi_setup().
# this will set up some fields like '_cffi_types', and only then
# it will invoke the chained list of functions that will really
# build (notably) the constant objects, as <cdata> if they are
# pointers, and store them as attributes on the 'library' object.
class FFILibrary(object):
_cffi_python_module = module
_cffi_ffi = self.ffi
_cffi_dir = []
def __dir__(self):
return FFILibrary._cffi_dir + list(self.__dict__)
library = FFILibrary()
if module._cffi_setup(lst, VerificationError, library):
import warnings
warnings.warn("reimporting %r might overwrite older definitions" %
(self.verifier.get_module_name()))
#
# finally, call the loaded_cpy_xxx() functions. This will perform
# the final adjustments, like copying the Python->C wrapper
# functions from the module to the 'library' object, and setting
# up the FFILibrary class with properties for the global C variables.
self._load(module, 'loaded', library=library)
module._cffi_original_ffi = self.ffi
module._cffi_types_of_builtin_funcs = self._types_of_builtin_functions
return library
def setImage(width, height, image_data):
global _imageData
imp.acquire_lock()
_imageWidth = width
_imageHeight = height
_imageData = bytearray(image_data)
imp.release_lock()
def view_import(name, globals={}, locals={}, fromlist=[], level=-1):
"""the drop-in replacement for __import__, that optionally imports
locally as well.
"""
# don't override nested imports
save_import = __builtin__.__import__
__builtin__.__import__ = local_import
if imp.lock_held():
# this is a side-effect import, don't do it remotely, or even
# ignore the local effects
return local_import(name, globals, locals, fromlist, level)
imp.acquire_lock()
if local:
mod = local_import(name, globals, locals, fromlist, level)
else:
raise NotImplementedError(
"remote-only imports not yet implemented")
imp.release_lock()
key = name + ':' + ','.join(fromlist or [])
if level == -1 and key not in modules:
modules.add(key)
if fromlist:
print "importing %s from %s on engine(s)" % (
','.join(fromlist), name)
else:
print "importing %s on engine(s)" % name
results.append(
self.apply_async(remote_import, name, fromlist, level))
# restore override
__builtin__.__import__ = save_import
return mod
def get_led_data():
led_data_copy = bytearray()
if _ledData:
imp.acquire_lock()
led_data_copy = bytearray(_ledData)
imp.release_lock()
return led_data_copy
def _populate(self):
"""
Stub method - this base class does no auto-loading.
"""
"""
Fill in all the cache information. This method is threadsafe, in the
sense that every caller will see the same state upon return, and if the
cache is already initialised, it does no work.
"""
if self.loaded:
return
if not self.loads_installed:
self.loaded = True
return
# Note that we want to use the import lock here - the app loading is
# in many cases initiated implicitly by importing, and thus it is
# possible to end up in deadlock when one thread initiates loading
# without holding the importer lock and another thread then tries to
# import something which also launches the app loading. For details of
# this situation see #18251.
imp.acquire_lock()
try:
if self.loaded:
return
for app_name in settings.INSTALLED_APPS:
if app_name in self.handled:
continue
self.load_app(app_name, True)
if not self.nesting_level:
for app_name in self.postponed:
self.load_app(app_name)
self.loaded = True
finally:
imp.release_lock()
def get_image_data():
img_data_copy = bytearray()
if _imageData:
imp.acquire_lock()
img_data_copy = bytearray(_imageData)
imp.release_lock()
return (_imageWidth, _imageHeight, img_data_copy)
def blocking(fullname, requested_type):
if utils.is_reactor_thread():
try:
raise ImportError(
"Cannot import %s: Remote import from reactor thread." %
fullname)
except:
logger.exception(
"Cannot import %s: Remote import from reactor thread." %
fullname)
raise
def in_reactor():
d = receiver(fullname, requested_type)
return d
try:
release_count = 0
while imp.lock_held():
release_count += 1
imp.release_lock()
return threads.blockingCallFromThread(reactor, in_reactor)
finally:
while release_count:
release_count -= 1
imp.acquire_lock()
def check_and_unload(self):
# Look through currently loaded modules:
for name, module in sys.modules.copy().items():
# Look only at the modules not in the the whitelist:
if name not in self.whitelist and hasattr(module, '__file__'):
# Only consider modules which are .py files, no C extensions or builtin
# modules:
if module.__file__ is None:
continue
module_file = module.__file__
if module_file.endswith('.pyc'):
module_file = os.path.splitext(module_file)[0] + '.py'
if not module_file.endswith('.py') or not os.path.exists(
module_file):
continue
# Check and store the modified time of the .py file:
modified_time = os.path.getmtime(module_file)
previous_modified_time = self.modified_times.setdefault(
name, modified_time)
self.modified_times[name] = modified_time
if modified_time != previous_modified_time:
# A module has been modified! Unload all modules
# not in the whitelist:
message = '%s modified: all modules will be reloaded next run.\n' % module_file
sys.stderr.write(message)
if self.debug:
print("ModuleWatcher: whitelist is:")
for name in sorted(self.whitelist):
print(" " + name)
print("\nModuleWatcher: modules unloaded:")
# Acquire the import lock so that we don't unload
# modules whilst an import is in progess:
imp.acquire_lock()
try:
for name in sorted(sys.modules):
if name not in self.whitelist:
# This unloads a module. This is slightly
# more general than reload(module), but
# has the same caveats regarding existing
# references. This also means that any
# exception in the import will occur later,
# once the module is (re)imported, rather
# than now where catching the exception
# would have to be handled differently.
del sys.modules[name]
if name in self.modified_times:
del self.modified_times[name]
if self.debug:
print(" " + name)
# Replace sys.meta_path with the cached whitelist,
# effectively removing all since-added entries from
# it. Replacement is done in-place in case other code
# holds references to sys.meta_path, and to preserve
# order, since order is relevant.
sys.meta_path[:] = self.meta_whitelist
finally:
# We're done mucking around with the cached
# modules, normal imports in other threads
# may resume:
imp.release_lock()
def load_module(self, fullname):
"""
Make sure that loading module is thread-safe using aq_method_lock to make
sure that modules do not disappear because of an ongoing reset
"""
# In Python < 3.3, the import lock is a global lock for all modules:
# http://bugs.python.org/issue9260
#
# So, release the import lock acquired by import statement on all hooks to
# load objects from ZODB. When an object is requested from ZEO, it sends a
# RPC request and lets the asyncore thread gets the reply. This reply may
# be a tuple (PICKLE, TID), sent directly to the first thread, or an
# Exception, which tries to import a ZODB module and thus creates a
# deadlock because of the global import lock
#
# Also, handle the case where find_module() may be called without import
# statement as it does not change anything in sys.modules
import_lock_held = True
try:
imp.release_lock()
except RuntimeError:
import_lock_held = False
aq_method_lock.acquire()
try:
return self.__load_module(fullname)
finally:
aq_method_lock.release()
# Internal release of import lock at the end of import machinery will
# fail if the hook is not acquired
if import_lock_held:
imp.acquire_lock()
def build(self, source, filename):
imp.acquire_lock()
try:
d = self.get(filename)
if d is not None:
return d
base, ext = os.path.splitext(filename)
name = os.path.join(self.path, base + ".py")
log.debug("writing source to disk (%d bytes)." % len(source))
fd, fn = tempfile.mkstemp(prefix=base, suffix='.tmp', dir=self.path)
temp = os.fdopen(fd, 'w')
try:
try:
temp.write("%s\n" % '# -*- coding: utf-8 -*-')
temp.write(source)
finally:
temp.close()
except:
os.remove(fn)
raise
os.rename(fn, name)
log.debug("compiling %s into byte-code..." % filename)
py_compile.compile(name)
return self._load(base, name)
finally:
imp.release_lock()
def load(moduleName, searchPath, applicationDomain):
''' Dynamically load a plugin module knowing its name and path.
Return the module object.
'''
# Lock the import global mutex
imp.acquire_lock()
try:
# Just try to return something from the registry. If it fails then we must actually load the module.
try:
return _registry[applicationDomain][moduleName]
except KeyError:
pass
# Check module name to guard against directory traversal and other annoyances.
if _regexModuleName.search(moduleName) == None:
ImportError(_('Invalid plugin name {0}.').format(moduleName))
searchPath = os.path.realpath(searchPath)
# Try to find the relevant module. If an exception arises, let the caller handle it.
moduleObject = imp.load_source(moduleName, searchPath + '/' + moduleName + '.py')
# Register the plugin locally (we don't want to rely solely on the global modules namespace)
if applicationDomain not in _registry:
_registry[applicationDomain] = {}
_registry[applicationDomain][moduleName] = moduleObject
return moduleObject
finally:
# Release import global mutex
imp.release_lock()
def importer():
imp.acquire_lock()
sys.modules[fake_module_name] = partial_module
import_started.set()
time.sleep(0.01) # Give the other thread time to try and acquire.
sys.modules[fake_module_name] = complete_module
imp.release_lock()
def load_plugins():
for dirs in get_dir_list(pwd):
full_path_dir = os.path.join(pwd, dirs)
plugin_dirs.append(full_path_dir)
plugins = []
for plugin_dir in plugin_dirs:
try:
for f in os.listdir(plugin_dir):
if f.endswith(".py") and f != "__init__.py":
plugins.append((f[:-3], plugin_dir))
except OSError:
print "Failed to access: %s" % plugin_dir
continue
fh = None
for (name, dirs) in plugins:
try:
acquire_lock()
fh, filename, desc = find_module(name, [dirs])
old = sys.modules.get(name)
if old is not None:
del sys.modules[name]
load_module(name, fh, filename, desc)
finally:
if fh:
fh.close()
release_lock()
def load_module(self, fullname, path=None):
imp.acquire_lock()
try:
# PEP302 If there is an existing module object named 'fullname'
# in sys.modules, the loader must use that existing module.
module = sys.modules.get(fullname)
if module is None:
filename = pyi_os_path.os_path_join(sys.prefix, fullname + self._suffix)
fp = open(filename, 'rb')
module = imp.load_module(fullname, fp, filename, self._c_ext_tuple)
# Set __file__ attribute.
if hasattr(module, '__setattr__'):
module.__file__ = filename
else:
# Some modules (eg: Python for .NET) have no __setattr__
# and dict entry have to be set.
module.__dict__['__file__'] = filename
except Exception:
# Remove 'fullname' from sys.modules if it was appended there.
if fullname in sys.modules:
sys.modules.pop(fullname)
# Release the interpreter's import lock.
imp.release_lock()
raise # Raise the same exception again.
# Release the interpreter's import lock.
imp.release_lock()
return module
def build(self, source, filename):
imp.acquire_lock()
try:
base, ext = os.path.splitext(filename)
name = os.path.join(self.path, base + ".py")
log.debug("writing source to disk (%d bytes)." % len(source))
temp = tempfile.NamedTemporaryFile(
prefix=base, suffix='.tmp', dir=self.path, delete=False)
try:
try:
temp.write("%s\n" % '# -*- coding: utf-8 -*-')
temp.write(source)
finally:
temp.close()
except:
os.remove(temp.name)
raise
os.rename(temp.name, name)
log.debug("compiling %s into byte-code..." % filename)
py_compile.compile(name)
return self._load(base, name)
finally:
imp.release_lock()
def load_module(self, fullname):
imp.acquire_lock()
try:
#print "loading module %s"%fullname
if fullname in sys.modules:
return sys.modules[fullname]
mod = None
c = None
if self.extension == "py":
mod = imp.new_module(fullname)
mod.__name__ = fullname
mod.__file__ = ("<memimport>/%s" % self.path).replace("/", sep)
mod.__loader__ = self
if self.is_pkg:
mod.__path__ = [mod.__file__.rsplit(sep, 1)[0]]
mod.__package__ = fullname
else:
mod.__package__ = fullname.rsplit('.', 1)[0]
sys.modules[fullname] = mod
code = compile(self.contents, mod.__file__, "exec")
exec code in mod.__dict__
elif self.extension in ["pyc", "pyo"]:
mod = imp.new_module(fullname)
mod.__name__ = fullname
mod.__file__ = ("<memimport>/%s" % self.path).replace("/", sep)
mod.__loader__ = self
if self.is_pkg:
mod.__path__ = [mod.__file__.rsplit(sep, 1)[0]]
mod.__package__ = fullname
else:
mod.__package__ = fullname.rsplit('.', 1)[0]
sys.modules[fullname] = mod
c = marshal.loads(self.contents[8:])
exec c in mod.__dict__
elif self.extension in ("dll", "pyd", "so"):
initname = "init" + fullname.rsplit(".", 1)[-1]
path = fullname.replace(".", "/") + "." + self.extension
#print "Loading %s from memory"%fullname
#print "init:%s, %s.%s"%(initname,fullname,self.extension)
mod = _memimporter.import_module(self.contents, initname,
fullname, path)
mod.__name__ = fullname
mod.__file__ = ("<memimport>/%s" % self.path).replace("/", sep)
mod.__loader__ = self
mod.__package__ = fullname.rsplit('.', 1)[0]
sys.modules[fullname] = mod
except Exception as e:
if fullname in sys.modules:
del sys.modules[fullname]
import traceback
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback)
print "PupyPackageLoader: Error while loading package %s (%s) : %s" % (
fullname, self.extension, str(e))
raise e
finally:
imp.release_lock()
mod = sys.modules[
fullname] # reread the module in case it changed itself
return mod
def other_thread():
imp.acquire_lock() # 3
assert imp.lock_held()
lock_held.release() # 4
test_complete.acquire() # 7
imp.release_lock() # 8
lock_released.release() # 9
def load_module(self, fullname):
"""
Make sure that loading module is thread-safe using aq_method_lock to make
sure that modules do not disappear because of an ongoing reset
"""
# In Python < 3.3, the import lock is a global lock for all modules:
# http://bugs.python.org/issue9260
#
# So, release the import lock acquired by import statement on all hooks to
# load objects from ZODB. When an object is requested from ZEO, it sends a
# RPC request and lets the asyncore thread gets the reply. This reply may
# be a tuple (PICKLE, TID), sent directly to the first thread, or an
# Exception, which tries to import a ZODB module and thus creates a
# deadlock because of the global import lock
#
# Also, handle the case where find_module() may be called without import
# statement as it does not change anything in sys.modules
import_lock_held = True
try:
imp.release_lock()
except RuntimeError:
import_lock_held = False
aq_method_lock.acquire()
try:
return self.__load_module(fullname)
finally:
aq_method_lock.release()
# Internal release of import lock at the end of import machinery will
# fail if the hook is not acquired
if import_lock_held:
imp.acquire_lock()
def import_pyv8():
# Importing non-existing modules is a bit tricky in Python:
# if we simply call `import PyV8` and module doesn't exists,
# Python will cache this failed import and will always
# throw exception even if this module appear in PYTHONPATH.
# To prevent this, we have to manually test if
# PyV8.py(c) exists in PYTHONPATH before importing PyV8
if 'PyV8' in sys.modules and 'PyV8' not in globals():
# PyV8 was loaded by ST2, create global alias
globals()['PyV8'] = __import__('PyV8')
return
loaded = False
f, pathname, description = imp.find_module('PyV8')
bin_f, bin_pathname, bin_description = imp.find_module('_PyV8')
if f:
try:
imp.acquire_lock()
globals()['_PyV8'] = imp.load_module('_PyV8', bin_f, bin_pathname, bin_description)
globals()['PyV8'] = imp.load_module('PyV8', f, pathname, description)
imp.release_lock()
loaded = True
finally:
# Since we may exit via an exception, close fp explicitly.
if f:
f.close()
if bin_f:
bin_f.close()
if not loaded:
raise ImportError('No PyV8 module found')
def __import__(*args, **kwargs):
"""
Normally python protects imports against concurrency by doing some locking
at the C level (at least, it does that in CPython). This function just
wraps the normal __import__ functionality in a recursive lock, ensuring that
we're protected against greenlet import concurrency as well.
"""
if len(args) > 0 and not issubclass(type(args[0]),
allowed_module_name_types):
# if a builtin has been acquired as a bound instance method,
# python knows not to pass 'self' when the method is called.
# No such protection exists for monkey-patched builtins,
# however, so this is necessary.
args = args[1:]
imp.acquire_lock()
try:
try:
_g_import_lock.acquire()
except RuntimeError:
# we've seen this under PyPy, a recursion error
# importing 'platform'
return _import(*args, **kwargs)
try:
result = _import(*args, **kwargs)
finally:
_g_import_lock.release()
finally:
imp.release_lock()
return result
