def setImage(width, height, image_data):
global _imageData
imp.acquire_lock()
_imageWidth = width
_imageHeight = height
_imageData = bytearray(image_data)
imp.release_lock()
def import_pyv8():
# Importing non-existing modules is a bit tricky in Python:
# if we simply call `import PyV8` and module doesn't exists,
# Python will cache this failed import and will always
# throw exception even if this module appear in PYTHONPATH.
# To prevent this, we have to manually test if
# PyV8.py(c) exists in PYTHONPATH before importing PyV8
if 'PyV8' in sys.modules and 'PyV8' not in globals():
# PyV8 was loaded by ST2, create global alias
globals()['PyV8'] = __import__('PyV8')
return
loaded = False
f, pathname, description = imp.find_module('PyV8')
bin_f, bin_pathname, bin_description = imp.find_module('_PyV8')
if f:
try:
imp.acquire_lock()
globals()['_PyV8'] = imp.load_module('_PyV8', bin_f, bin_pathname, bin_description)
globals()['PyV8'] = imp.load_module('PyV8', f, pathname, description)
imp.release_lock()
loaded = True
finally:
# Since we may exit via an exception, close fp explicitly.
if f:
f.close()
if bin_f:
bin_f.close()
if not loaded:
raise ImportError('No PyV8 module found')
def load_plugin(plugin_file, valid_base_class):
"""
Load a plugin into memory and extract its single interface class.
"""
# construct a uuid and use this as the module name to ensure
# that each import is unique
import uuid
module_uid = uuid.uuid4().hex
module = None
try:
imp.acquire_lock()
module = imp.load_source(module_uid, plugin_file)
except Exception:
# dump out the callstack for this one -- to help people get good messages when there is a plugin error
(exc_type, exc_value, exc_traceback) = sys.exc_info()
message = ""
message += "Failed to load plugin %s. The following error was reported:\n" % plugin_file
message += "Exception: %s - %s\n" % (exc_type, exc_value)
message += "Traceback (most recent call last):\n"
message += "\n".join( traceback.format_tb(exc_traceback))
raise TankError(message)
finally:
imp.release_lock()
# cool, now validate the module
found_classes = list()
introspection_error_reported = None
try:
for var in dir(module):
value = getattr(module, var)
if isinstance(value, type) and issubclass(value, valid_base_class) and value != valid_base_class:
found_classes.append(value)
except Exception, e:
introspection_error_reported = str(e)
def _run_module_code(code, init_globals=None,
mod_name=None, mod_fname=None,
mod_loader=None, alter_sys=False):
"""Helper for run_module"""
# Set up the top level namespace dictionary
if alter_sys:
# Modify sys.argv[0] and sys.module[mod_name]
temp_module = imp.new_module(mod_name)
mod_globals = temp_module.__dict__
saved_argv0 = sys.argv[0]
restore_module = mod_name in sys.modules
if restore_module:
saved_module = sys.modules[mod_name]
imp.acquire_lock()
try:
sys.argv[0] = mod_fname
sys.modules[mod_name] = temp_module
try:
_run_code(code, mod_globals, init_globals,
mod_name, mod_fname, mod_loader)
finally:
sys.argv[0] = saved_argv0
if restore_module:
sys.modules[mod_name] = saved_module
else:
del sys.modules[mod_name]
finally:
imp.release_lock()
# Copy the globals of the temporary module, as they
# may be cleared when the temporary module goes away
return mod_globals.copy()
else:
# Leave the sys module alone
return _run_code(code, {}, init_globals,
mod_name, mod_fname, mod_loader)
def other_thread():
imp.acquire_lock() # 3
assert imp.lock_held()
lock_held.release() # 4
test_complete.acquire() # 7
imp.release_lock() # 8
lock_released.release() # 9
def dump_main(ctx):
ctx.stats.timer('main', 'started')
ret = True
if len(ctx.routes.groups()) < 2:
log.error("There is only one group in route list: {0}. "
"sdc recovery could not be made."
.format(ctx.routes.groups()))
return False
try:
ctx.merged_filename = lookup_keys(ctx)
except KeyboardInterrupt:
log.error("Caught Ctrl+C. Terminating.")
ctx.stats.timer('main', 'finished')
return False
log.debug("Merged_filename: %s, address: %s, groups: %s, tmp_dir:%s",
ctx.merged_filename, ctx.address, ctx.groups, ctx.tmp_dir)
if ctx.dry_run:
return ret
if ctx.custom_recover == '':
recover(ctx)
else:
import imp
log.debug("Loading module: {0}".format(ctx.custom_recover))
imp.acquire_lock()
custom_recover = imp.load_source('custom_recover', ctx.custom_recover)
imp.release_lock()
custom_recover.recover(ctx)
ctx.stats.timer('main', 'finished')
return ret
def load_module(self, fullname):
if fullname in sys.modules:
return sys.modules[fullname]
path = self.cache[fullname]
# must insert to sys.modules first, to avoid cyclic imports
try:
imp.acquire_lock()
mod = sys.modules[fullname] = imp.new_module(fullname)
try:
if path is None:
mod.__file__ = "<namespace module>"
mod.__path__ = []
else:
info = imp.find_module(os.path.basename(path), [os.path.dirname(path)])
mod = imp.load_module(fullname, *info)
# replace the stub in sys.modules
sys.modules[fullname] = mod
except Exception:
del sys.modules[fullname]
raise
finally:
imp.release_lock()
return mod
def get_led_data():
led_data_copy = bytearray()
if _ledData:
imp.acquire_lock()
led_data_copy = bytearray(_ledData)
imp.release_lock()
return led_data_copy
def load_module(self, fullname):
"""
Make sure that loading module is thread-safe using aq_method_lock to make
sure that modules do not disappear because of an ongoing reset
"""
# In Python < 3.3, the import lock is a global lock for all modules:
# http://bugs.python.org/issue9260
#
# So, release the import lock acquired by import statement on all hooks to
# load objects from ZODB. When an object is requested from ZEO, it sends a
# RPC request and lets the asyncore thread gets the reply. This reply may
# be a tuple (PICKLE, TID), sent directly to the first thread, or an
# Exception, which tries to import a ZODB module and thus creates a
# deadlock because of the global import lock
#
# Also, handle the case where find_module() may be called without import
# statement as it does not change anything in sys.modules
import_lock_held = True
try:
imp.release_lock()
except RuntimeError:
import_lock_held = False
aq_method_lock.acquire()
try:
return self.__load_module(fullname)
finally:
aq_method_lock.release()
# Internal release of import lock at the end of import machinery will
# fail if the hook is not acquired
if import_lock_held:
imp.acquire_lock()
def __init__(self, to=None, defaultPk=None, **kwargs):
super(ForeignObjectValidatorMixin, self).__init__(
to=to, defaultPk=defaultPk, **kwargs)
self._defaultPk = defaultPk
if to is None:
return
if 'instance' in kwargs:
# Replace _to method to foreign constructor
if isinstance(to, string_types):
import sys
to_path = to.split('.')
object_rel = to_path.pop()
package_rel = '.'.join(to_path)
if PY2:
import imp as _imp
else:
import _imp
_imp.acquire_lock()
module1 = __import__('.'.join(to_path))
_imp.release_lock()
try:
self._to = getattr(sys.modules[package_rel], object_rel)
except AttributeError:
raise AttributeError('Package "%s" not contain model %s' %
(package_rel, object_rel))
else:
self._to = to
def oss_to_epoch(timestamp):
# given a time string like: 2014-05-15T11:18:32.000Z
# convert it to an epoch time.
imp.acquire_lock()
ts = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.000Z").timetuple()
imp.release_lock()
return (int)(time.mktime(ts))
def get_image_data():
img_data_copy = bytearray()
if _imageData:
imp.acquire_lock()
img_data_copy = bytearray(_imageData)
imp.release_lock()
return (_imageWidth, _imageHeight, img_data_copy)
def _get_loader(mod_name, path=None):
"""Retrieve a PEP 302 loader for the given module or package
If the module or package is accessible via the normal import
mechanism, a wrapper around the relevant part of that machinery
is returned.
Non PEP 302 mechanisms (e.g. the Windows registry) used by the
standard import machinery to find files in alternative locations
are partially supported, but are searched AFTER sys.path. Normally,
these locations are searched BEFORE sys.path, preventing sys.path
entries from shadowing them.
For this to cause a visible difference in behaviour, there must
be a module or package name that is accessible via both sys.path
and one of the non PEP 302 file system mechanisms. In this case,
the emulation will find the former version, while the builtin
import mechanism will find the latter.
Items of the following types can be affected by this discrepancy:
imp.C_EXTENSION
imp.PY_SOURCE
imp.PY_COMPILED
imp.PKG_DIRECTORY
"""
try:
loader = sys.modules[mod_name].__loader__
except (KeyError, AttributeError):
loader = None
if loader is None:
imp.acquire_lock()
try:
# Module not in sys.modules, or uses an unhooked loader
parts = mod_name.rsplit(".", 1)
if len(parts) == 2:
# Sub package, so use parent package's path
pkg_name, sub_name = parts
if pkg_name and pkg_name[0] != ".":
if path is not None:
raise ImportError("Path argument must be None " "for a dotted module name")
pkg = _get_package(pkg_name)
try:
path = pkg.__path__
except AttributeError:
raise ImportError(pkg_name + " is not a package")
else:
raise ImportError("Relative import syntax is not " "supported by _get_loader()")
else:
# Top level module, so stick with default path
sub_name = mod_name
for importer in sys.meta_path:
loader = importer.find_module(mod_name, path)
if loader is not None:
# Found a metahook to handle the module
break
else:
# Handling via the standard path mechanism
loader = _get_path_loader(mod_name, path)
finally:
imp.release_lock()
return loader
def build(self, source, filename):
imp.acquire_lock()
try:
d = self.get(filename)
if d is not None:
return d
base, ext = os.path.splitext(filename)
name = os.path.join(self.path, base + ".py")
log.debug("writing source to disk (%d bytes)." % len(source))
fd, fn = tempfile.mkstemp(prefix=base, suffix='.tmp', dir=self.path)
temp = os.fdopen(fd, 'w')
try:
try:
temp.write("%s\n" % '# -*- coding: utf-8 -*-')
temp.write(source)
finally:
temp.close()
except:
os.remove(fn)
raise
os.rename(fn, name)
log.debug("compiling %s into byte-code..." % filename)
py_compile.compile(name)
return self._load(base, name)
finally:
imp.release_lock()
def loadPlugins(self):
"""Load plugins by iterating files in plugin directories.
"""
plugins = []
for dir in self.directories:
try:
for f in os.listdir(dir):
if f.endswith(".py") and f != "__init__.py":
plugins.append((f[:-3], dir))
except OSError:
print "Failed to access: %s" % dir
continue
fh = None
mod = None
for (name, dir) in plugins:
try:
acquire_lock()
fh, filename, desc = find_module(name, [dir])
old = sys.modules.get(name)
if old is not None:
# make sure we get a fresh copy of anything we are trying
# to load from a new path
del sys.modules[name]
mod = load_module(name, fh, filename, desc)
finally:
if fh:
fh.close()
release_lock()
if hasattr(mod, "__all__"):
attrs = [getattr(mod, x) for x in mod.__all__]
for plug in attrs:
if not issubclass(plug, Plugin):
continue
self._loadPlugin(plug())
def _load_plugin(self, plugin_id, path=None):
manifest = self.manifests[plugin_id]
path = path or manifest.src_path or self.plugin_path(plugin_id)
imp.acquire_lock()
try:
finder = PluginModuleFinder(path)
sys.meta_path.append(finder)
main_module = manifest.main_module or plugin_id
result = importlib.import_module(main_module)
plugin_exports = [main_module]
plugin_modules = []
if manifest.exported_modules:
plugin_exports.extend(manifest.exported_modules)
for module_name in finder.plugin_modules:
if hasattr(sys.modules[module_name], '__path__'):
pkg_prefix = module_name + '.'
should_remove = not any(
name.startswith(pkg_prefix) for name in plugin_exports)
else:
should_remove = module_name not in plugin_exports
if should_remove:
sys.modules.pop(module_name, None)
else:
plugin_modules.append(module_name)
self.module_refcount[module_name] += 1
self.plugin_modules[plugin_id] = plugin_modules
return result
finally:
sys.meta_path.remove(finder)
imp.release_lock()
def view_import(name, globals={}, locals={}, fromlist=[], level=0):
"""the drop-in replacement for __import__, that optionally imports
locally as well.
"""
# don't override nested imports
save_import = builtin_mod.__import__
builtin_mod.__import__ = local_import
if imp.lock_held():
# this is a side-effect import, don't do it remotely, or even
# ignore the local effects
return local_import(name, globals, locals, fromlist, level)
imp.acquire_lock()
if local:
mod = local_import(name, globals, locals, fromlist, level)
else:
raise NotImplementedError("remote-only imports not yet implemented")
imp.release_lock()
key = name+':'+','.join(fromlist or [])
if level <= 0 and key not in modules:
modules.add(key)
if not quiet:
if fromlist:
print("importing %s from %s on engine(s)"%(','.join(fromlist), name))
else:
print("importing %s on engine(s)"%name)
results.append(self.apply_async(remote_import, name, fromlist, level))
# restore override
builtin_mod.__import__ = save_import
return mod
def find_module(self, name, path):
# Ignore anything not in the form kaa.foo, or if kaa.foo is an egg in sys.path.
self.discover_kaa_eggs()
if not name.startswith('kaa.') or name.count('.') > 1 or name in self.kaa_eggs:
return
kaa_base_path = __file__.rsplit('/', 1)[0]
name = name[4:].replace('.', '/')
imp.acquire_lock()
try:
# Scenario 1: kaa.base is an on-disk tree (egg or otherwise)
if not self.zip:
try:
return KaaLoader(imp.find_module(name, [kaa_base_path]))
except ImportError:
pass
# Scenario 2: kaa.base is a zipped egg
if '.egg/' in kaa_base_path and self.zip is not False:
try:
if not self.zip:
self.zip = zipimport.zipimporter(kaa_base_path)
except ImportError:
# Not a valid zipped egg, cache the result so we don't try again.
self.zip = False
else:
if self.zip.find_module('kaa.base.' + name):
# kaa.base.foo found inside egg.
return KaaLoader(self.zip)
finally:
imp.release_lock()
def build(self, source, filename):
imp.acquire_lock()
try:
base, ext = os.path.splitext(filename)
name = os.path.join(self.path, base + ".py")
log.debug("writing source to disk (%d bytes)." % len(source))
temp = tempfile.NamedTemporaryFile(
prefix=base, suffix='.tmp', dir=self.path, delete=False)
try:
try:
temp.write("%s\n" % '# -*- coding: utf-8 -*-')
temp.write(source)
finally:
temp.close()
except:
os.remove(temp.name)
raise
os.rename(temp.name, name)
log.debug("compiling %s into byte-code..." % filename)
py_compile.compile(name)
return self._load(base, name)
finally:
imp.release_lock()
def find_module(self, fullname, path=None):
dbg("find_module (%s): %r" % (self.dir, fullname), end='')
acquire_lock()
try:
dir = self.dir
try:
files = sys.quickimport_cache[dir]
except Exception, e:
raise ImportError("Can't import %r: No quickimport dir cache for dir %r: %s" % (fullname, dir, e) )
basename = fullname.rsplit('.', 1)[-1]
basenameNormcase = os.path.normcase(basename)
if not basenameNormcase in files:
for s in suffixes:
if (basenameNormcase + s) in files:
break
else:
dbg("")
return None
# this path is a candidate
importer = sys.path_importer_cache.get(dir)
assert importer is self
try:
dbg("testing.. ", end='')
loader = ImpLoader(fullname, *find_module(basename, [dir]))
dbg("found")
return loader
except ImportError, e:
dbg(e)
return None
def __init__(self, module_names=()):
"""
:param module_names: iterator of module names
"""
self._saved = {}
imp.acquire_lock()
self.save(*module_names)
def testLock(self):
LOOPS = 50
# The import lock may already be held, e.g. if the test suite is run
# via "import test.autotest".
lock_held_at_start = imp.lock_held()
self.verify_lock_state(lock_held_at_start)
for i in range(LOOPS):
imp.acquire_lock()
self.verify_lock_state(True)
for i in range(LOOPS):
imp.release_lock()
# The original state should be restored now.
self.verify_lock_state(lock_held_at_start)
if not lock_held_at_start:
try:
imp.release_lock()
except RuntimeError:
pass
else:
self.fail("release_lock() without lock should raise "
"RuntimeError")
def load_module(self, fullname, path=None):
imp.acquire_lock()
try:
# PEP302 If there is an existing module object named 'fullname'
# in sys.modules, the loader must use that existing module.
module = sys.modules.get(fullname)
if module is None:
filename = pyi_os_path.os_path_join(sys.prefix, fullname + self._suffix)
fp = open(filename, 'rb')
module = imp.load_module(fullname, fp, filename, self._c_ext_tuple)
# Set __file__ attribute.
if hasattr(module, '__setattr__'):
module.__file__ = filename
else:
# Some modules (eg: Python for .NET) have no __setattr__
# and dict entry have to be set.
module.__dict__['__file__'] = filename
except Exception:
# Remove 'fullname' from sys.modules if it was appended there.
if fullname in sys.modules:
sys.modules.pop(fullname)
# Release the interpreter's import lock.
imp.release_lock()
raise # Raise the same exception again.
# Release the interpreter's import lock.
imp.release_lock()
return module
def find_module(self, fullname, path=None):
"""
PEP-302 finder.find_module() method for the ``sys.meta_path`` hook.
fullname fully qualified name of the module
path None for a top-level module, or package.__path__ for submodules or subpackages.
Return a loader object if the module was found, or None if it wasn't. If find_module() raises
an exception, it will be propagated to the caller, aborting the import.
"""
# Acquire the interpreter's import lock for the current thread. Tis
# lock should be used by import hooks to ensure thread-safety when
# importing modules.
imp.acquire_lock()
# TODO rewrite this method.
module_loader = None # None means - no module found in this importer.
try:
print fullname
if fullname in self._pyz_archive.toc:
print '... found'
# Tell the import machinery to use self.load_module() to load the module.
module_loader = self
else:
print '... found'
finally:
# Release the interpreter's import lock.
imp.release_lock()
return module_loader
def importer():
imp.acquire_lock()
sys.modules[fake_module_name] = partial_module
import_started.set()
time.sleep(0.01) # Give the other thread time to try and acquire.
sys.modules[fake_module_name] = complete_module
imp.release_lock()
def __import__(*args, **kwargs):
"""
Normally python protects imports against concurrency by doing some locking
at the C level (at least, it does that in CPython). This function just
wraps the normal __import__ functionality in a recursive lock, ensuring that
we're protected against greenlet import concurrency as well.
"""
if len(args) > 0 and not issubclass(type(args[0]), _allowed_module_name_types):
# if a builtin has been acquired as a bound instance method,
# python knows not to pass 'self' when the method is called.
# No such protection exists for monkey-patched builtins,
# however, so this is necessary.
args = args[1:]
# TODO: It would be nice not to have to acquire the locks
# if the module is already imported (in sys.modules), but the interpretation
# of the arguments is somewhat complex
imp.acquire_lock()
try:
_g_import_lock.acquire()
try:
result = _import(*args, **kwargs)
finally:
_g_import_lock.release()
finally:
imp.release_lock()
return result
def load(moduleName, searchPath, applicationDomain):
''' Dynamically load a plugin module knowing its name and path.
Return the module object.
'''
# Lock the import global mutex
imp.acquire_lock()
try:
# Just try to return something from the registry. If it fails then we must actually load the module.
try:
return _registry[applicationDomain][moduleName]
except KeyError:
pass
# Check module name to guard against directory traversal and other annoyances.
if _regexModuleName.search(moduleName) == None:
ImportError(_('Invalid plugin name {0}.').format(moduleName))
searchPath = os.path.realpath(searchPath)
# Try to find the relevant module. If an exception arises, let the caller handle it.
moduleObject = imp.load_source(moduleName, searchPath + '/' + moduleName + '.py')
# Register the plugin locally (we don't want to rely solely on the global modules namespace)
if applicationDomain not in _registry:
_registry[applicationDomain] = {}
_registry[applicationDomain][moduleName] = moduleObject
return moduleObject
finally:
# Release import global mutex
imp.release_lock()
def load_plugins():
for dirs in get_dir_list(pwd):
full_path_dir = os.path.join(pwd, dirs)
plugin_dirs.append(full_path_dir)
plugins = []
for plugin_dir in plugin_dirs:
try:
for f in os.listdir(plugin_dir):
if f.endswith(".py") and f != "__init__.py":
plugins.append((f[:-3], plugin_dir))
except OSError:
print "Failed to access: %s" % plugin_dir
continue
fh = None
for (name, dirs) in plugins:
try:
acquire_lock()
fh, filename, desc = find_module(name, [dirs])
old = sys.modules.get(name)
if old is not None:
del sys.modules[name]
load_module(name, fh, filename, desc)
finally:
if fh:
fh.close()
release_lock()
def _populate(self):
"""
Fill in all the cache information. This method is threadsafe, in the
sense that every caller will see the same state upon return, and if the
cache is already initialised, it does no work.
"""
if self.loaded:
return
# Note that we want to use the import lock here - the app loading is
# in many cases initiated implicitly by importing, and thus it is
# possible to end up in deadlock when one thread initiates loading
# without holding the importer lock and another thread then tries to
# import something which also launches the app loading. For details of
# this situation see #18251.
imp.acquire_lock()
try:
if self.loaded:
return
for app_name in settings.INSTALLED_APPS:
if app_name in self.handled:
continue
self.load_app(app_name, True)
if not self.nesting_level:
for app_name in self.postponed:
self.load_app(app_name)
self.loaded = True
finally:
imp.release_lock()
def load(self, name, notify=True):
"""
Load a plugin.
"""
if name in self.plugins:
self.unload(name)
try:
if name in self.modules:
imp.acquire_lock()
module = imp.reload(self.modules[name])
imp.release_lock()
else:
file, filename, info = imp.find_module(name, [plugins_dir, default_plugin_path])
imp.acquire_lock()
module = imp.load_module(name, file, filename, info)
imp.release_lock()
except Exception as e:
import traceback
log.debug("Could not load plugin: \n%s", traceback.format_exc())
self.core.information("Could not load plugin: %s" % e, 'Error')
return
finally:
if imp.lock_held():
imp.release_lock()
self.modules[name] = module
self.commands[name] = {}
self.keys[name] = {}
self.tab_keys[name] = {}
self.tab_commands[name] = {}
self.event_handlers[name] = []
self.plugins[name] = module.Plugin(self, self.core, plugins_conf_dir)
if notify:
self.core.information('Plugin %s loaded' % name, 'Info')
def __load_module(self):
try:
self.__module = sys.modules.get(self.__hook_module_name, None)
if self.__module is None:
imp.acquire_lock()
self.__module = sys.modules.get(self.__hook_module_name, None)
if self.__module is not None:
imp.release_lock()
return
added_paths = self.__add_plugin_paths()
try:
self.__module = imp.load_source(self.__hook_module_name,
self.__module_path)
finally:
self.__remove_plugin_paths(added_paths)
imp.release_lock()
except Exception as e:
raise HandledError('Failed to load hook module {}. {}'.format(
self.__module_path, traceback.format_exc()))
def handleInvocation(self,daemon,conn):
ver,body,pflags = self.receiveMsg(conn)
if not body:
# something went wrong even before receiving the full message body
return
if ver!=self.version:
Log.error('PYROAdapter','incompatible protocol version')
self.returnException(conn, ProtocolError('incompatible protocol version'))
return
# Unpickle the request, which is a tuple:
# (object ID, method name, flags, (arg1,arg2,...))
importer=fromlist=None
try:
try:
# install a custom importer to intercept any extra needed modules
# when unpickling the request just obtained from the client
imp.acquire_lock()
importer=agent_import(__builtin__.__import__)
__builtin__.__import__=importer
req=self._unpickleRequest(pflags, body)
if type(req)!=tuple or len(req)!=4 or type(req[3])!=tuple:
# sanity check failed
raise ProtocolError("invalid request data format")
finally:
__builtin__.__import__=importer.orig_import
imp.release_lock()
except ImportError,x:
if Pyro.config.PYRO_MOBILE_CODE:
# return a special exception that will be processed by client;
# it will call the internal 'remote_supply_code' member
if importer:
modname=importer.name
fromlist=importer.fromlist
else:
modname = x.args[0][16:]
fromlist=None
self.returnException(conn, _InternalNoModuleError(modname,fromlist),0) # don't shutdown!
else:
Log.error('PYROAdapter','code problem with incoming object: '+str(x))
self.returnException(conn, NoModuleError(* x.args))
return
def load_plugins(master):
"""
Find and load all plugins
:return:
"""
last_error['root'] = master
imp.acquire_lock()
internal = []
for name in os.listdir(config.internal_plugin_dir):
if name.endswith('.py') and not name[0] in ['.', '_']:
try:
plugin = Plugin(name[:-3],
os.path.join(config.internal_plugin_dir, name))
plugin.folder = None # Suppress listing in Plugins prefs tab
internal.append(plugin)
except:
print_exc()
PLUGINS.extend(
sorted(internal, key=lambda p: operator.attrgetter('name')(p).lower()))
found = []
for name in os.listdir(config.plugin_dir):
if name[0] in ['.', '_']:
pass
elif name.endswith('.disabled'):
name, discard = name.rsplit('.', 1)
found.append(Plugin(name, None))
else:
try:
# Add plugin's folder to Python's load path in case plugin has dependencies.
sys.path.append(os.path.join(config.plugin_dir, name))
found.append(
Plugin(name,
os.path.join(config.plugin_dir, name, 'load.py')))
except:
print_exc()
PLUGINS.extend(
sorted(found, key=lambda p: operator.attrgetter('name')(p).lower()))
imp.release_lock()
def check_and_unload(self):
# Look through currently loaded modules:
for name, module in sys.modules.copy().items():
# Look only at the modules not in the the whitelist:
if name not in self.whitelist and hasattr(module, '__file__'):
# Only consider modules which are .py files, no C extensions:
module_file = module.__file__.replace('.pyc', '.py')
if not module_file.endswith('.py') or not os.path.exists(
module_file):
continue
# Check and store the modified time of the .py file:
modified_time = os.path.getmtime(module_file)
previous_modified_time = self.modified_times.setdefault(
name, modified_time)
self.modified_times[name] = modified_time
if modified_time != previous_modified_time:
# A module has been modified! Unload all modules
# not in the whitelist:
message = '%s modified: all modules will be reloaded next run.\n' % module_file
sys.stderr.write(message)
# Acquire the import lock so that we don't unload
# modules whilst an import is in progess:
imp.acquire_lock()
try:
for name in sys.modules.copy():
if name not in self.whitelist:
# This unloads a module. This is slightly
# more general than reload(module), but
# has the same caveats regarding existing
# references. This also means that any
# exception in the import will occur later,
# once the module is (re)imported, rather
# than now where catching the exception
# would have to be handled differently.
del sys.modules[name]
if name in self.modified_times:
del self.modified_times[name]
finally:
# We're done mucking around with the cached
# modules, normal imports in other threads
# may resume:
imp.release_lock()
def gnrImport(source, importAs=None, avoidDup=False):
"""add???
:param source: add???
:param importAs: add???. Default value is ``None``
:param avoidDup: if ``True``, allow to avoid duplicates. Default value is ``False``
:returns: add???
"""
modkey = source
path_sep = os.path.sep
if path_sep in source:
if avoidDup and not importAs:
importAs = os.path.splitext(source)[0].replace(path_sep, '_').replace('.', '_')
modkey = importAs or os.path.splitext(os.path.basename(source))[0]
try:
m = sys.modules[modkey]
return m
except KeyError:
pass
path = None
if path_sep in source:
path = [os.path.dirname(source)]
source = os.path.splitext(os.path.basename(source))[0]
segments = source.split('.')
error = None
for segment in segments:
module_file = None
try:
imp.acquire_lock()
module_file, module_path, module_description = imp.find_module(segment, path)
except ImportError:
imp.release_lock()
return None
if importAs and segment == segments[-1]:
segment = importAs
try:
module = imp.load_module(segment, module_file, module_path, module_description)
path = getattr(module, '__path__', None)
except Exception, e:
module = None
error = e
finally:
def run(self):
while True:
if self.queue.empty():
break
code = self.queue.get()
# processing the item
# time.sleep(code)
n = 0
imp.acquire_lock()
end = datetime.datetime.strptime(str(datetime.date.today()),
'%Y-%m-%d')
imp.release_lock()
while n < 730:
n += 1
print(self.name, code, end)
socket.setdefaulttimeout(100)
try:
df = ts.get_tick_data(code, end.date())
if type(
df
) != types.NoneType and df.index.size > 0 and 'alert' not in df.values[
0][0]:
self.tick_retry_count = 0
self.download_fenshi_tocsv(code, end.date(), df)
# self.download_fenshi_todb(code, end.date(), df)
else:
print(' ----------no data')
end -= datetime.timedelta(days=1)
except Exception as e:
print('get stock fenbi failed! code = [' + str(code) +
'],date = [' + str(end.date()) +
'] ,retruCount = [' + str(self.tick_retry_count) +
'] , Error = [' + str(e) + ']')
self.get_tick_data(code, end)
end -= datetime.timedelta(days=1)
print('queue 大小:', self.queue.qsize())
self.queue.task_done()
return
def load_module(self, name):
fullname = 'kaa.base.' + name[4:]
if fullname in sys.modules:
return sys.modules[fullname]
zipped = isinstance(self._info, zipimport.zipimporter)
imp.acquire_lock()
try:
mod = imp.load_module(
fullname, *
self._info) if not zipped else self._info.load_module(fullname)
# Cache loaded module as both kaa.foo (for external access) and
# kaa.base.foo (for internal access by kaa.base code).
sys.modules[name] = sys.modules[fullname] = mod
finally:
if not zipped and self._info[0]:
self._info[0].close()
imp.release_lock()
return mod
def load_plugins():
"""
Load all found plugins
:return:
"""
found = find_plugins()
imp.acquire_lock()
for plugname in found:
try:
with open(found[plugname], "rb") as plugfile:
plugmod = imp.load_module(plugname, plugfile, found[plugname],
(".py", "r", imp.PY_SOURCE))
if "plugin_start" in dir(plugmod):
plugmod.plugin_start()
PLUGINS[plugname] = plugmod
except Exception as plugerr:
sys.stderr.write('%s\n' % plugerr) # appears in %TMP%/EDMarketConnector.log in packaged Windows app
imp.release_lock()
def load_module(self, fullname, path=None):
imp.acquire_lock()
try:
# PEP302 If there is an existing module object named 'fullname'
# in sys.modules, the loader must use that existing module.
module = sys.modules.get(fullname)
if module is None:
module = imp.init_builtin(fullname)
except Exception:
# Remove 'fullname' from sys.modules if it was appended there.
if fullname in sys.modules:
sys.modules.pop(fullname)
raise # Raise the same exception again.
finally:
# Release the interpreter's import lock.
imp.release_lock()
return module
def get_app(self, app_label, emptyOK=False):
"""
Returns the module containing the models for the given app_label. If
the app has no models in it and 'emptyOK' is True, returns None.
"""
self._populate()
imp.acquire_lock()
try:
for app_name in settings.INSTALLED_APPS:
if app_label == app_name.split('.')[-1]:
mod = self.load_app(app_name, False)
if mod is None:
if emptyOK:
return None
raise ImproperlyConfigured("App with label %s is missing a models.py module." % app_label)
else:
return mod
raise ImproperlyConfigured("App with label %s could not be found" % app_label)
finally:
imp.release_lock()
def geth(tk,myfile,date):
try :
h=ord(myfile.read(1))
m=ord(myfile.read(1))
s=ord(myfile.read(1))
ms=ord(myfile.read(1))*10
if (ms==0):
imp.acquire_lock()
tk.time=str(h+100)[1:3]+':'+str(m+100)[1:3]+':'+str(s+100)[1:3]
tk.datetime=datetime.strptime(' '.join([date, tk.time]), '%Y%m%d %H:%M:%S')
imp.release_lock()
# print 'date1' ,self.tk.datetime
else:
imp.acquire_lock()
tk.time=str(h+100)[1:3]+':'+str(m+100)[1:3]+':'+str(s+100)[1:3]+'.'+str(ms)
tk.datetime=datetime.strptime(' '.join([date, tk.time]), '%Y%m%d %H:%M:%S.%f')
imp.release_lock()
# print 'date2',self.tk.datetime
except TypeError:
pass
def register_import_hook(name, callable):
imp.acquire_lock()
try:
hooks = _import_hooks.get(name, None)
if not name in _import_hooks or hooks is None:
# If no entry in registry or entry already flagged with
# None then module may have been loaded, in which case
# need to check and fire hook immediately.
hooks = _import_hooks.get(name)
module = sys.modules.get(name, None)
if module is not None:
# The module has already been loaded so fire hook
# immediately.
_import_hooks[name] = None
callable(module)
else:
# No hook has been registered so far so create list
# and add current hook.
_import_hooks[name] = [callable]
else:
# Hook has already been registered, so append current
# hook.
_import_hooks[name].append(callable)
finally:
imp.release_lock()
def load_plugins(master):
"""
Find and load all plugins
"""
last_error['root'] = master
imp.acquire_lock()
internal = []
for name in os.listdir(config.internal_plugin_dir):
if name.endswith('.py') and not name[0] in ['.', '_']:
try:
plugin = Plugin(name[:-3], os.path.join(config.internal_plugin_dir, name))
plugin.folder = None # Suppress listing in Plugins prefs tab
internal.append(plugin)
except:
print_exc()
PLUGINS.extend(sorted(internal, key = lambda p: operator.attrgetter('name')(p).lower()))
# Add plugin folder to load path so packages can be loaded from plugin folder
sys.path.append(config.plugin_dir)
found = []
# Load any plugins that are also packages first
for name in sorted(os.listdir(config.plugin_dir),
key = lambda n: (not os.path.isfile(os.path.join(config.plugin_dir, n, '__init__.py')), n.lower())):
if not os.path.isdir(os.path.join(config.plugin_dir, name)) or name[0] in ['.', '_']:
pass
elif name.endswith('.disabled'):
name, discard = name.rsplit('.', 1)
found.append(Plugin(name, None))
else:
try:
# Add plugin's folder to load path in case plugin has internal package dependencies
sys.path.append(os.path.join(config.plugin_dir, name))
found.append(Plugin(name, os.path.join(config.plugin_dir, name, 'load.py')))
except:
print_exc()
PLUGINS.extend(sorted(found, key = lambda p: operator.attrgetter('name')(p).lower()))
imp.release_lock()
def apply(self, library_name, inputhash, dry_run=False):
"""
Changes the state of the system according to what the module
specified does
Keyword arguments:
library_name -- the python module to load
inputhash -- the list of dictionaries of config for the library
dry_run -- whether or not to actually change the system
"""
log.debug('entering state.apply %s',
[library_name, inputhash, dry_run])
if library_name not in sys.modules:
try:
imp.acquire_lock()
mod = imp.find_module(library_name, self.libraries)
imp.load_module(library_name, *mod)
except ImportError:
log.exception("Couldn't find module %s in dirs %s",
library_name, self.libraries)
raise
finally:
if imp.lock_held():
imp.release_lock()
log.debug('loading library: %s', library_name)
library = importlib.import_module(library_name)
schema = library.schema()
for item in inputhash:
jsonschema.validate(item, schema)
failed = library.verify(inputhashes=inputhash, log=log)
log.debug('dry run: %s', dry_run)
if not dry_run:
log.debug('applying state...')
library.apply(inputhashes=failed, log=log)
failed = library.verify(inputhashes=inputhash, log=log)
if len(failed) > 0:
log.error("Failed for good on {}".format(failed))
log.debug('Leaving state.apply %s', failed)
return failed
def loadPlugins(self):
"""Load plugins by iterating files in plugin directories.
"""
plugins = []
#print '********Directory directories:',self.directories
for dir in self.directories:
try:
for f in os.listdir(dir):
if f.endswith(".py") and f != "__init__.py":
plugins.append((f[:-3], dir))
except OSError:
security_warning("Faild to load Plugin")
#print "Failed to access: %s" % dir
continue
fh = None
mod = None
#print '********Directory all plugins:',plugins
for (name, dir) in plugins:
try:
acquire_lock()
fh, filename, desc = find_module(name, [dir])
#print '********Directory fh,filename,desc:',fh,filename,desc,name
old = sys.modules.get(name)
if old is not None:
# make sure we get a fresh copy of anything we are trying
# to load from a new path
del sys.modules[name]
mod = load_module(name, fh, filename, desc)
finally:
if fh:
fh.close()
release_lock()
if hasattr(mod, "__all__"):
#print '********Directory mod __all__:',mod.__all__
attrs = [getattr(mod, x) for x in mod.__all__]
#print '********Directory attrs:',attrs
for plug in attrs:
if not issubclass(plug, Plugin):
continue
self._loadPlugin(plug())
def remote_supply_code(self, name, module, sourceaddr):
# XXX this is nasty code, and also duplicated in protocol.py _retrieveCode()
if Pyro.config.PYRO_MOBILE_CODE and self.codeValidator(name,module,sourceaddr):
try:
imp.acquire_lock() # threadsafe imports
if name in sys.modules and getattr(sys.modules[name],'_PYRO_bytecode',None):
# already have this module, don't import again
# we checked for the _PYRO_bytecode attribute because that is only
# present when all loading code below completed successfully
return
Log.msg('ObjBase','loading supplied code: ',name,'from',str(sourceaddr))
if module[0:4]!=imp.get_magic():
# compile source code
code=compile(module,'<downloaded>','exec')
else:
# read bytecode from the client
code=marshal.loads(module[8:])
# make the module hierarchy and add all names to sys.modules
name=name.split('.')
path=''
mod=new.module("pyro-agent-context")
for m in name:
path+='.'+m
# use already loaded modules instead of overwriting them
real_path = path[1:]
if sys.modules.has_key(real_path):
mod = sys.modules[real_path]
else:
setattr(mod,m,new.module(path[1:]))
mod=getattr(mod,m)
sys.modules[path[1:]]=mod
# execute the module code in the right module.
exec code in mod.__dict__
# store the bytecode for possible later reference if we need to pass it on
mod.__dict__['_PYRO_bytecode'] = module
finally:
imp.release_lock()
else:
Log.warn('ObjBase','attempt to supply code denied: ',name,'from',str(sourceaddr))
raise PyroError('attempt to supply code denied')
def add_module(self, name, code):
imp.acquire_lock()
try:
mod = imp.new_module(name)
sys.modules[name] = mod
try:
mod.__file__ = name + ".py"
exec code in mod.__dict__
mod.__loader__ = self
met_dbg('Executed code for: {0}'.format(name))
except e:
del sys.modules[name]
mod = None
except:
mod = None
finally:
imp.release_lock()
met_dbg('Result for {0}: {1}'.format(name, mod != None))
def load_module(name, path, log_import=True):
if log_import:
print('Loading module {} from {}.'.format(name, path))
result = None
imp.acquire_lock()
try:
fp, pathname, description = imp.find_module(name, [path])
try:
result = imp.load_module(name, fp, pathname, description)
finally:
if fp:
fp.close()
finally:
imp.release_lock()
return result
def test_manual_locking(self):
import thread, os, imp, time, sys
f = open(os.path.join(self.tmpdir, 'foobaz2.py'), 'w')
f.close() # empty
done = []
def f():
sys.path.insert(0, self.tmpdir)
import foobaz2
p = sys.path.pop(0)
assert p == self.tmpdir
done.append(1)
assert not imp.lock_held()
imp.acquire_lock()
assert imp.lock_held()
thread.start_new_thread(f, ())
time.sleep(0.9)
assert not done
assert imp.lock_held()
# check that it's a recursive lock
imp.acquire_lock()
assert imp.lock_held()
imp.acquire_lock()
assert imp.lock_held()
imp.release_lock()
assert imp.lock_held()
imp.release_lock()
assert imp.lock_held()
imp.release_lock()
assert not imp.lock_held()
self.waitfor(lambda: done)
assert done
def loadPlugins(self): # load的过程==（1）将文件名放入列表
"""Load plugins by iterating files in plugin directories.
"""
plugins = []
print ('********Directory directories:',self.directories)
for dir in self.directories:
try:
for f in os.listdir(dir):
if f.endswith(".py") and f != "__init__.py":
plugins.append((f[:-3], dir)) # [-n] 表示从后往前倒数n个数
except OSError:
print ("Failed to access: %s" % dir)
continue
fh = None
mod = None
print ('********Directory all plugins:',plugins)
for (name, dir) in plugins:
try:
acquire_lock()
fh, filename, desc = find_module(name, [dir]) ## 根据目录和文件名找到模块
print ('********Directory fh,filename,desc:',fh,filename,desc,name)
old = sys.modules.get(name)
if old is not None:
# make sure we get a fresh copy of anything we are trying
# to load from a new path
del sys.modules[name]
mod = load_module(name, fh, filename, desc) # 得到一个module对象
finally:
if fh:
fh.close()
release_lock()
if hasattr(mod, "__all__"):
print ('********Directory mod __all__:',mod.__all__)
attrs = [getattr(mod, x) for x in mod.__all__] # getattr() 获取某个对象的属性值
print ('********Directory attrs:',attrs)
for plug in attrs:
if not issubclass(plug, Plugin):
continue
self._loadPlugin(plug())
def load_plugins_from(plugin_dir, package, config):
# import in sorted order to let it be predictable; lets user plugins import
# pieces of other plugins imported earlier
plugins = []
errors = []
Plugin = package.Plugin
for f in sorted(os.listdir(plugin_dir)):
if f == '__init__.py' or not f.endswith(".py"):
continue
mname = f[:-3]
qualifiedname = package.__name__ + '.' + mname
imp.acquire_lock()
try:
module = imp.load_source(qualifiedname,
os.path.join(plugin_dir, f))
sys.modules[qualifiedname] = module
setattr(package, mname, module)
except Exception, e: # pylint: disable=W0703
errors.append(
PluginError(mname, "Failed to add plugin '%s'" % mname, e))
continue
finally:
def CsLoadMemoryData(self, name, filename, data, ispackage=False):
if data[2:6] != imp.get_magic():
raise ImportError('Bad magic number in %s' % filename)
code = marshal.loads(data[10:])
imp.acquire_lock() # Required in threaded applications
try:
mod = imp.new_module(name)
sys.modules[name] = mod # To handle circular and submodule imports
# it should come before exec.
try:
mod.__file__ = filename # Is not so important.
# For package you have to set mod.__path__ here.
# Here I handle simple cases only.
if ispackage:
mod.__path__ = [name.replace('.', '/')]
exec code in mod.__dict__
except:
del sys.modules[name]
raise
finally:
imp.release_lock()
return mod
def fork_with_import_lock(level):
release = 0
in_child = False
try:
try:
for i in range(level):
imp.acquire_lock()
release += 1
pid = os.fork()
in_child = not pid
finally:
for i in range(release):
imp.release_lock()
except RuntimeError:
if in_child:
if verbose > 1:
print("RuntimeError in child")
os._exit(1)
raise
if in_child:
os._exit(0)
self.wait_impl(pid)
def __init__(self):
"""
Load, unzip and initialize the Zip archive bundled with the executable.
"""
# Examine all items in sys.path and the one like /path/executable_name?117568
# is the correct executable with bundled zip archive. Use this value
# for the ZlibArchive class and remove this item from sys.path.
# It was needed only for FrozenImporter class. Wrong path from sys.path
# Raises ArchiveReadError exception.
for pyz_filepath in sys.path:
# We need to acquire the interpreter's import lock here
# because ZlibArchive() seeks through and reads from the
# zip archive.
imp.acquire_lock()
try:
# Unzip zip archive bundled with the executable.
self._pyz_archive = ZlibArchive(pyz_filepath)
# Verify the integrity of the zip archive with Python modules.
self._pyz_archive.checkmagic()
# End this method since no Exception was raised we can assume
# ZlibArchive was successfully loaded. Let's remove 'pyz_filepath'
# from sys.path.
sys.path.remove(pyz_filepath)
# Some runtime hook might need access to the list of available
# frozen module. Let's make them accessible as a set().
self.toc = set(self._pyz_archive.toc.keys())
# Return - no error was raised.
return
except IOError:
# Item from sys.path is not ZlibArchive let's try next.
continue
except ArchiveReadError:
# Item from sys.path is not ZlibArchive let's try next.
continue
finally:
imp.release_lock()
# sys.path does not contain filename of executable with bundled zip archive.
# Raise import error.
raise ImportError("Can't load frozen modules.")
def _run_module_code(code,
init_globals=None,
mod_name=None,
mod_fname=None,
mod_loader=None,
alter_sys=False):
"""Helper for run_module"""
# Set up the top level namespace dictionary
if alter_sys:
# Modify sys.argv[0] and sys.module[mod_name]
temp_module = imp.new_module(mod_name)
mod_globals = temp_module.__dict__
saved_argv0 = sys.argv[0]
restore_module = mod_name in sys.modules
if restore_module:
saved_module = sys.modules[mod_name]
imp.acquire_lock()
try:
sys.argv[0] = mod_fname
sys.modules[mod_name] = temp_module
try:
_run_code(code, mod_globals, init_globals, mod_name, mod_fname,
mod_loader)
finally:
sys.argv[0] = saved_argv0
if restore_module:
sys.modules[mod_name] = saved_module
else:
del sys.modules[mod_name]
finally:
imp.release_lock()
# Copy the globals of the temporary module, as they
# may be cleared when the temporary module goes away
return mod_globals.copy()
else:
# Leave the sys module alone
return _run_code(code, {}, init_globals, mod_name, mod_fname,
mod_loader)
def _loadModule(module):
oldGA = LazyModule.__getattribute__
oldSA = LazyModule.__setattr__
modGA = ModuleType.__getattribute__
modSA = ModuleType.__setattr__
LazyModule.__getattribute__ = modGA
LazyModule.__setattr__ = modSA
acquire_lock()
try:
try:
# don't reload if already loaded!
if module.__dict__.keys() == ['__name__']:
# Get Python to do the real import!
reload(module)
try:
for hook in getModuleHooks(module.__name__):
hook(module)
finally:
# Ensure hooks are not called again, even if they fail
postLoadHooks[module.__name__] = None
except:
# Reset our state so that we can retry later
if '__file__' not in module.__dict__:
LazyModule.__getattribute__ = oldGA.im_func
LazyModule.__setattr__ = oldSA.im_func
raise
try:
# Convert to a real module (if under 2.2)
module.__class__ = ModuleType
except TypeError:
pass # 2.3 will fail, but no big deal
finally:
release_lock()
def _handle_P1Alert(self, P1Alert):
switch_dpid = P1Alert.switch_dpid
self.host_id = P1Alert.host_id
imp.acquire_lock()
now_time = time.strftime("%H:%M:%S", time.localtime())
imp.release_lock()
log.info("sy receive P1Alert %s" % now_time)
global INTERVAL, request_trigger
if not request_trigger:
# 防止重复使用timer_func 只有当trigger 为 False的时候允许进入
request_trigger = True
# 在周期性执行core.Phase2.timer_func前先发一条
self.send_stats_request(switch_dpid)
Timer(INTERVAL,
core.Phase2.timer_func,
recurring=True,
args=(switch_dpid, ))
pass
def compile_module(self):
"""Builds a temporary module from the script file
:raises exceptions.IOError: if the compilation of the script module failed
"""
try:
imp.acquire_lock()
code = compile(self.script,
'%s (%s)' % (self.filename, self._script_id),
'exec')
# load module
module_name = os.path.splitext(self.filename)[0] + str(
self._script_id)
tmp_module = imp.new_module(module_name)
exec(code, tmp_module.__dict__)
# return the module
self.compiled_module = tmp_module
except Exception as e:
self.compiled_module = None
raise
finally:
imp.release_lock()
def get_app(self, app_label, emptyOK=False):
"""
Returns the module containing the models for the given app_label.
Returns None if the app has no models in it and emptyOK is True.
Raises UnavailableApp when set_available_apps() in in effect and
doesn't include app_label.
"""
self._populate()
imp.acquire_lock()
try:
for app_name in settings.INSTALLED_APPS:
if app_label == app_name.split('.')[-1]:
mod = self.load_app(app_name, False)
if mod is None and not emptyOK:
raise ImproperlyConfigured("App with label %s is missing a models.py module." % app_label)
if self.available_apps is not None and app_label not in self.available_apps:
raise UnavailableApp("App with label %s isn't available." % app_label)
return mod
raise ImproperlyConfigured("App with label %s could not be found" % app_label)
finally:
imp.release_lock()
def dump_main(ctx):
global g_ctx
g_ctx = ctx
ctx.monitor.stats.timer('main', 'started')
ret = True
if len(ctx.routes.groups()) < 2:
log.error("There is only one group in route list: {0}. "
"sdc recovery could not be made."
.format(ctx.routes.groups()))
return False
try:
ctx.merged_filename = lookup_keys(ctx)
except KeyboardInterrupt:
log.error("Caught Ctrl+C. Terminating.")
ctx.monitor.stats.timer('main', 'finished')
return False
log.debug("Merged_filename: %s, address: %s, groups: %s, tmp_dir:%s",
ctx.merged_filename, ctx.address, ctx.groups, ctx.tmp_dir)
if ctx.dry_run:
return ret
if ctx.custom_recover == '':
from ..dc_recovery import recover
recover(ctx)
else:
import imp
log.debug("Loading module: {0}".format(ctx.custom_recover))
imp.acquire_lock()
custom_recover = imp.load_source('custom_recover', ctx.custom_recover)
imp.release_lock()
custom_recover.recover(ctx)
ctx.monitor.stats.timer('main', 'finished')
return ret
