def test_streamer_linecount():
util.log_to_stderr(util.SUBDEBUG)
s = Streamer(processes=2)
stats = s.consume(range(10))
assert len(stats) == 5
# 0 is skipped.
assert stats[PROCESSING_TOTAL] == 10
assert stats[PROCESSING_SUCCESS] == 9
assert stats[PROCESSING_SKIPPED] == 1
assert stats[PROCESSING_ERROR] == 0
def prepare(data):
"""
Try to get current process ready to unpickle process object
"""
old_main_modules.append(sys.modules['__main__'])
if 'name' in data:
process.current_process().name = data['name']
if 'authkey' in data:
process.current_process()._authkey = data['authkey']
if 'log_to_stderr' in data and data['log_to_stderr']:
util.log_to_stderr()
if 'log_level' in data:
util.get_logger().setLevel(data['log_level'])
if 'sys_path' in data:
sys.path = data['sys_path']
if 'sys_argv' in data:
sys.argv = data['sys_argv']
if 'dir' in data:
os.chdir(data['dir'])
if 'orig_dir' in data:
process.ORIGINAL_DIR = data['orig_dir']
if 'main_path' in data:
main_path = data['main_path']
main_name = os.path.splitext(os.path.basename(main_path))[0]
main_name = main_name == '__init__' and os.path.basename(os.path.dirname(main_path))
if main_name != 'ipython':
import imp
if main_path is None:
dirs = None
elif os.path.basename(main_path).startswith('__init__.py'):
dirs = [os.path.dirname(os.path.dirname(main_path))]
else:
dirs = [os.path.dirname(main_path)]
if not main_name not in sys.modules:
raise AssertionError(main_name)
file, path_name, etc = imp.find_module(main_name, dirs)
try:
main_module = imp.load_module('__parents_main__', file, path_name, etc)
finally:
if file:
file.close()
sys.modules['__main__'] = main_module
main_module.__name__ = '__main__'
for obj in main_module.__dict__.values():
try:
if obj.__module__ == '__parents_main__':
obj.__module__ = '__main__'
except Exception:
pass
return
def log_to_stderr(level=None):
"""
Turn on logging and add a handler which prints to stderr
"""
from multiprocessing.util import log_to_stderr
return log_to_stderr(level)
def prepare(data):
'''
Try to get current process ready to unpickle process object
'''
if 'name' in data:
process.current_process().name = data['name']
if 'authkey' in data:
process.current_process().authkey = data['authkey']
if 'log_to_stderr' in data and data['log_to_stderr']:
util.log_to_stderr()
if 'log_level' in data:
util.get_logger().setLevel(data['log_level'])
if 'log_fmt' in data:
import logging
util.get_logger().handlers[0].setFormatter(
logging.Formatter(data['log_fmt'])
)
if 'sys_path' in data:
sys.path = data['sys_path']
if 'sys_argv' in data:
sys.argv = data['sys_argv']
if 'dir' in data:
os.chdir(data['dir'])
if 'orig_dir' in data:
process.ORIGINAL_DIR = data['orig_dir']
if hasattr(mp, 'set_start_method'):
mp.set_start_method('loky', force=True)
if 'tacker_pid' in data:
from . import semaphore_tracker
semaphore_tracker._semaphore_tracker._pid = data["tracker_pid"]
if 'init_main_from_name' in data:
_fixup_main_from_name(data['init_main_from_name'])
elif 'init_main_from_path' in data:
_fixup_main_from_path(data['init_main_from_path'])
def setup_logger(loglevel=conf.CELERYD_LOG_LEVEL, logfile=None,
format=conf.CELERYD_LOG_FORMAT, **kwargs):
"""Setup the ``multiprocessing`` logger. If ``logfile`` is not specified,
``stderr`` is used.
Returns logger object.
"""
logger = get_default_logger(loglevel=loglevel)
if logger.handlers:
# Logger already configured
return logger
if logfile:
handler = logging.FileHandler
if hasattr(logfile, "write"):
handler = logging.StreamHandler
loghandler = handler(logfile)
formatter = logging.Formatter(format)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
else:
from multiprocessing.util import log_to_stderr
log_to_stderr()
return logger
def log_to_stderr(level=None):
from multiprocessing.util import log_to_stderr
return log_to_stderr(level)
def log_to_stderr(level=None):
'''
Turn on logging and add a handler which prints to stderr
'''
from multiprocessing.util import log_to_stderr
return log_to_stderr(level)
if os.path.islink(logfile):
sys.exit('Logfile %s is a symlink. Exiting..' % logfile)
try:
logging.basicConfig(filename=logfile, level=level, format='%(asctime)s %(levelname)s %(funcName)s:%(lineno)d %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
os.chmod(logfile, 0600)
except IOError as (errno, strerror):
if errno == int('13'):
sys.exit('Error while writing to logfile: %s' % strerror)
try:
database = Database('yamls/', includes)
if len(database.issues) == 0:
sys.exit('Empty database. Exiting..')
# stderr to /dev/null
devnull_fd = open(os.devnull, "w")
sys.stderr = devnull_fd
log_to_stderr()
# Starts the asynchronous workers. Amount of workers is the same as cores in server.
# http://docs.python.org/library/multiprocessing.html#multiprocessing.pool.multiprocessing.Pool
logging.debug('Starting workers.')
pool = Pool()
pool.apply_async(Worker, [arguments['--home'], post_process])
# Starts the actual populator daemon to get possible locations, which will be verified by workers.
# http://docs.python.org/library/multiprocessing.html#multiprocessing.Process
p = PopulateScanQueue()
if arguments['-r']:
logging.debug('Scanning recursively from path: %s', arguments['-r'])
populator = Process(target=p.populate, args=([arguments['-r']],))
elif arguments['--home']:
logging.debug('Scanning predefined variables: %s', arguments['--home'])
populator = Process(target=p.populate_predefined, args=(arguments['--home'], arguments['--check-modes'],))
else:
def log_to_stderr(level = None):
from multiprocessing.util import log_to_stderr
return log_to_stderr(level)
def prepare(data):
'''
Try to get current process ready to unpickle process object
'''
old_main_modules.append(sys.modules['__main__'])
if 'name' in data:
process.current_process().name = data['name']
if 'authkey' in data:
process.current_process()._authkey = data['authkey']
if 'log_to_stderr' in data and data['log_to_stderr']:
util.log_to_stderr()
if 'log_level' in data:
util.get_logger().setLevel(data['log_level'])
if 'sys_path' in data:
sys.path = data['sys_path']
if 'sys_argv' in data:
sys.argv = data['sys_argv']
if 'dir' in data:
os.chdir(data['dir'])
if 'orig_dir' in data:
process.ORIGINAL_DIR = data['orig_dir']
if 'main_path' in data:
main_path = data['main_path']
main_name = os.path.splitext(os.path.basename(main_path))[0]
if main_name == '__init__':
main_name = os.path.basename(os.path.dirname(main_path))
if main_name != 'ipython':
import imp
if main_path is None:
dirs = None
elif os.path.basename(main_path).startswith('__init__.py'):
dirs = [os.path.dirname(os.path.dirname(main_path))]
else:
dirs = [os.path.dirname(main_path)]
assert main_name not in sys.modules, main_name
file, path_name, etc = imp.find_module(main_name, dirs)
try:
# We would like to do "imp.load_module('__main__', ...)"
# here. However, that would cause 'if __name__ ==
# "__main__"' clauses to be executed.
main_module = imp.load_module('__parents_main__', file,
path_name, etc)
finally:
if file:
file.close()
sys.modules['__main__'] = main_module
main_module.__name__ = '__main__'
# Try to make the potentially picklable objects in
# sys.modules['__main__'] realize they are in the main
# module -- somewhat ugly.
for obj in main_module.__dict__.values():
try:
if obj.__module__ == '__parents_main__':
obj.__module__ = '__main__'
except Exception:
pass
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 25 14:49:49 2018
@author: alexc
"""
import itertools
from scipy.special import comb
import math
import time, datetime
from multiprocessing import Pool, current_process
import multiprocessing.util as util
util.log_to_stderr(util.SUBDEBUG)
class Slicer():
def __init__(self, n, k, batch_size, folder_name):
self.n = n
self.k = k
self.combinations = itertools.combinations(range(n), k)
self.batch_size = batch_size
self.folder_name = folder_name
# self.results = []
self.max_util = 0
def test_batches(self, batch, slice_integer):
''' Test that the slice_combinations gets batches of the correct size '''
# Shows that we could use the slice integer to track the output
fname = self.folder_name + current_process().name + "_" + str(
slice_integer) + ".txt"
def prepare(data):
'''
Try to get current process ready to unpickle process object
'''
old_main_modules.append(sys.modules['__main__'])
if 'name' in data:
process.current_process().name = data['name']
if 'authkey' in data:
process.current_process()._authkey = data['authkey']
if 'log_to_stderr' in data and data['log_to_stderr']:
util.log_to_stderr()
if 'log_level' in data:
util.get_logger().setLevel(data['log_level'])
if 'sys_path' in data:
sys.path = data['sys_path']
if 'sys_argv' in data:
sys.argv = data['sys_argv']
if 'dir' in data:
os.chdir(data['dir'])
if 'orig_dir' in data:
process.ORIGINAL_DIR = data['orig_dir']
if 'main_path' in data:
# XXX (ncoghlan): The following code makes several bogus
# assumptions regarding the relationship between __file__
# and a module's real name. See PEP 302 and issue #10845
main_path = data['main_path']
main_name = os.path.splitext(os.path.basename(main_path))[0]
if main_name == '__init__':
main_name = os.path.basename(os.path.dirname(main_path))
if main_name == '__main__':
main_module = sys.modules['__main__']
main_module.__file__ = main_path
elif main_name != 'ipython':
# Main modules not actually called __main__.py may
# contain additional code that should still be executed
import imp
if main_path is None:
dirs = None
elif os.path.basename(main_path).startswith('__init__.py'):
dirs = [os.path.dirname(os.path.dirname(main_path))]
else:
dirs = [os.path.dirname(main_path)]
assert main_name not in sys.modules, main_name
sys.modules.pop('__mp_main__', None)
file, path_name, etc = imp.find_module(main_name, dirs)
try:
# We should not do 'imp.load_module("__main__", ...)'
# since that would execute 'if __name__ == "__main__"'
# clauses, potentially causing a psuedo fork bomb.
main_module = imp.load_module(
'__mp_main__', file, path_name, etc
)
finally:
if file:
file.close()
sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
def prepare(data):
'''
Try to get current process ready to unpickle process object
'''
old_main_modules.append(sys.modules['__main__'])
if 'name' in data:
process.current_process().name = data['name']
if 'authkey' in data:
process.current_process()._authkey = data['authkey']
if 'log_to_stderr' in data and data['log_to_stderr']:
util.log_to_stderr()
if 'log_level' in data:
util.get_logger().setLevel(data['log_level'])
if 'sys_path' in data:
sys.path = data['sys_path']
if 'sys_argv' in data:
sys.argv = data['sys_argv']
if 'dir' in data:
os.chdir(data['dir'])
if 'orig_dir' in data:
process.ORIGINAL_DIR = data['orig_dir']
if 'main_path' in data:
# XXX (ncoghlan): The following code makes several bogus
# assumptions regarding the relationship between __file__
# and a module's real name. See PEP 302 and issue #10845
# The problem is resolved properly in Python 3.4+, as
# described in issue #19946
main_path = data['main_path']
main_name = os.path.splitext(os.path.basename(main_path))[0]
if main_name == '__init__':
main_name = os.path.basename(os.path.dirname(main_path))
if main_name == '__main__':
# For directory and zipfile execution, we assume an implicit
# "if __name__ == '__main__':" around the module, and don't
# rerun the main module code in spawned processes
main_module = sys.modules['__main__']
main_module.__file__ = main_path
elif main_name != 'ipython':
# Main modules not actually called __main__.py may
# contain additional code that should still be executed
import imp
if main_path is None:
dirs = None
elif os.path.basename(main_path).startswith('__init__.py'):
dirs = [os.path.dirname(os.path.dirname(main_path))]
else:
dirs = [os.path.dirname(main_path)]
assert main_name not in sys.modules, main_name
file, path_name, etc = imp.find_module(main_name, dirs)
try:
# We would like to do "imp.load_module('__main__', ...)"
# here. However, that would cause 'if __name__ ==
# "__main__"' clauses to be executed.
main_module = imp.load_module('__parents_main__', file,
path_name, etc)
finally:
if file:
file.close()
sys.modules['__main__'] = main_module
main_module.__name__ = '__main__'
# Try to make the potentially picklable objects in
# sys.modules['__main__'] realize they are in the main
# module -- somewhat ugly.
for obj in main_module.__dict__.values():
try:
if obj.__module__ == '__parents_main__':
obj.__module__ = '__main__'
except Exception:
pass
def __del__(self):
#print("dtor subclass2:",super(),", type:",type(super()))
super(type(self), self).__del__()
class SubclassThree(BaseClass):
def __init__(self):
#print("ctor:",self)
pass
def __del__(self):
#print("dtor subclass3:",super(),", type:",type(super()))
pass
util.log_to_stderr(level=logging.DEBUG)
class Main():
def __init__(self):
self.manager = Manager()
self.first_q = self.manager.Queue()
self.second_q = self.manager.Queue()
self.objs = [SubclassOne(), SubclassTwo(), SubclassThree()]
for obj in self.objs:
self.first_q.put(obj)
#fruit = 'Apple'
#print('<<<<<<<<<<<<<')
#print(True if fruit == 'Apple' else False)
def prepare(data):
'''
Try to get current process ready to unpickle process object
'''
old_main_modules.append(sys.modules['__main__'])
if 'name' in data:
process.current_process().name = data['name']
if 'authkey' in data:
process.current_process()._authkey = data['authkey']
if 'log_to_stderr' in data and data['log_to_stderr']:
util.log_to_stderr()
if 'log_level' in data:
util.get_logger().setLevel(data['log_level'])
if 'sys_path' in data:
sys.path = data['sys_path']
if 'sys_argv' in data:
sys.argv = data['sys_argv']
if 'dir' in data:
os.chdir(data['dir'])
if 'orig_dir' in data:
process.ORIGINAL_DIR = data['orig_dir']
if 'main_path' in data:
main_path = data['main_path']
main_name = os.path.splitext(os.path.basename(main_path))[0]
if main_name == '__init__':
main_name = os.path.basename(os.path.dirname(main_path))
if main_name != 'ipython':
import imp
if main_path is None:
dirs = None
elif os.path.basename(main_path).startswith('__init__.py'):
dirs = [os.path.dirname(os.path.dirname(main_path))]
else:
dirs = [os.path.dirname(main_path)]
assert main_name not in sys.modules, main_name
file, path_name, etc = imp.find_module(main_name, dirs)
try:
# We would like to do "imp.load_module('__main__', ...)"
# here. However, that would cause 'if __name__ ==
# "__main__"' clauses to be executed.
main_module = imp.load_module(
'__parents_main__', file, path_name, etc
)
finally:
if file:
file.close()
sys.modules['__main__'] = main_module
main_module.__name__ = '__main__'
# Try to make the potentially picklable objects in
# sys.modules['__main__'] realize they are in the main
# module -- somewhat ugly.
for obj in main_module.__dict__.values():
try:
if obj.__module__ == '__parents_main__':
obj.__module__ = '__main__'
except Exception:
pass
def set_structures(datadict, scratch, procs=0):
"""
take dict of smiles->value and generate sdf from smiles.
Put in scratch/structures.sdf.gz
Put values in scratch/properties.{txt,npy}
"""
keys = datadict.keys()
results = []
# no mp
if procs == 0:
def get_results():
values = []
for key in keys:
values.append(datadict[key])
for smi, value in zip(keys, values):
result = prepare_sdf_and_csv(smi, value)
yield result
results = get_results()
# scale it out
elif procs > 0:
def workpackages():
for i, key in enumerate(keys):
smi = key
kelvin = datadict[key]
yield smi, kelvin
lines = workpackages()
import multiprocessing.util as util
util.log_to_stderr(util.SUBDEBUG)
p = Pool(procs)
results = p.map(prepare_sdf_and_csv_procs, lines)
print("wating for results")
fsdf = gzip.open(scratch + "structures.sdf.gz", 'w')
fprop = open(scratch + "properties.csv", 'w')
for i, result in enumerate(results):
if result is None: continue
molobj, values = result
mean = np.mean(values)
prtstr = np.round(values, decimals=1)
print("save {:4.2f}".format(mean), "-", prtstr)
sdfstr = cheminfo.molobj_to_sdfstr(molobj)
sdfstr += "$$$$\n"
fsdf.write(sdfstr.encode())
valuesstr = " ".join([str(x) for x in values])
# propstr = "{:} {:}\n".format(mean, standard_deviation)
propstr = f"{i} " + valuesstr + "\n"
fprop.write(propstr)
fsdf.close()
fprop.close()
return
def main(fd, verbose=0):
'''Run resource tracker.'''
# protect the process from ^C and "killall python" etc
if verbose:
util.log_to_stderr(level=util.DEBUG)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
if _HAVE_SIGMASK:
signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS)
for f in (sys.stdin, sys.stdout):
try:
f.close()
except Exception:
pass
if verbose:
util.debug("Main resource tracker is running")
registry = {rtype: dict() for rtype in _CLEANUP_FUNCS.keys()}
try:
# keep track of registered/unregistered resources
if sys.platform == "win32":
fd = msvcrt.open_osfhandle(fd, os.O_RDONLY)
with open(fd, 'rb') as f:
while True:
line = f.readline()
if line == b'': # EOF
break
try:
splitted = line.strip().decode('ascii').split(':')
# name can potentially contain separator symbols (for
# instance folders on Windows)
cmd, name, rtype = (
splitted[0], ':'.join(splitted[1:-1]), splitted[-1])
if cmd == 'PROBE':
continue
if rtype not in _CLEANUP_FUNCS:
raise ValueError(
f'Cannot register {name} for automatic cleanup: '
f'unknown resource type ({rtype}). Resource type '
'should be one of the following: '
f'{list(_CLEANUP_FUNCS.keys())}'
)
if cmd == 'REGISTER':
if name not in registry[rtype]:
registry[rtype][name] = 1
else:
registry[rtype][name] += 1
if verbose:
util.debug(
"[ResourceTracker] incremented refcount of "
f"{rtype} {name} "
f"(current {registry[rtype][name]})"
)
elif cmd == 'UNREGISTER':
del registry[rtype][name]
if verbose:
util.debug(
f"[ResourceTracker] unregister {name} {rtype}: "
f"registry({len(registry)})"
)
elif cmd == 'MAYBE_UNLINK':
registry[rtype][name] -= 1
if verbose:
util.debug(
"[ResourceTracker] decremented refcount of "
f"{rtype} {name} (current {registry[rtype][name]})"
)
if registry[rtype][name] == 0:
del registry[rtype][name]
try:
if verbose:
util.debug(
f"[ResourceTracker] unlink {name}"
)
_CLEANUP_FUNCS[rtype](name)
except Exception as e:
warnings.warn(
f'resource_tracker: {name}: {e!r}')
else:
raise RuntimeError(f'unrecognized command {cmd!r}')
except BaseException:
try:
sys.excepthook(*sys.exc_info())
except BaseException:
pass
finally:
# all processes have terminated; cleanup any remaining resources
def _unlink_resources(rtype_registry, rtype):
if rtype_registry:
try:
warnings.warn(
'resource_tracker: There appear to be '
f'{len(rtype_registry)} leaked {rtype} objects to '
'clean up at shutdown'
)
except Exception:
pass
for name in rtype_registry:
# For some reason the process which created and registered this
# resource has failed to unregister it. Presumably it has
# died. We therefore clean it up.
try:
_CLEANUP_FUNCS[rtype](name)
if verbose:
util.debug(f"[ResourceTracker] unlink {name}")
except Exception as e:
warnings.warn(f'resource_tracker: {name}: {e!r}')
for rtype, rtype_registry in registry.items():
if rtype == "folder":
continue
else:
_unlink_resources(rtype_registry, rtype)
# The default cleanup routine for folders deletes everything inside
# those folders recursively, which can include other resources tracked
# by the resource tracker). To limit the risk of the resource tracker
# attempting to delete twice a resource (once as part of a tracked
# folder, and once as a resource), we delete the folders after all
# other resource types.
if "folder" in registry:
_unlink_resources(registry["folder"], "folder")
if verbose:
util.debug("resource tracker shut down")
def parse_l(self, args):
""" Parse log-level and set it."""
log_to_stderr(args.loglevel)
def __init__(self, processes=None, initializer=None, initargs=(),
maxtasksperchild=None, timeout_seconds=30):
log_to_stderr(level=DEBUG)
self._setup_queues()
self._taskqueue = Queue.Queue()
self._cache = {}
self._state = RUN
self._maxtasksperchild = maxtasksperchild
self._initializer = initializer
self._initargs = initargs
self._proc_num = 0
if processes is None:
try:
processes = cpu_count()
except NotImplementedError:
processes = 1
if initializer is not None and not hasattr(initializer, '__call__'):
raise TypeError('initializer must be a callable')
self._processes = processes
self._pool = []
self._repopulate_pool()
self._worker_handler = threading.Thread(
target=Pool._handle_workers,
args=(self, )
)
self._worker_handler.daemon = True
self._worker_handler._state = RUN
self._worker_handler.start()
self._task_handler = threading.Thread(
target=Pool._handle_tasks,
args=(self._taskqueue, self._quick_put, self._outqueue, self._pool)
)
self._task_handler.daemon = True
self._task_handler._state = RUN
self._task_handler.start()
self.timeout_seconds = timeout_seconds
self._result_handler = threading.Thread(
target=TimeoutPool._handle_results,
args=(self._outqueue, self._quick_get, self._cache, self._pool, self.timeout_seconds)
)
self._result_handler.daemon = True
self._result_handler._state = RUN
self._result_handler.start()
self._terminate = Finalize(
self, self._terminate_pool,
args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
self._worker_handler, self._task_handler,
self._result_handler, self._cache),
exitpriority=15
)
level=level,
format=
'%(asctime)s %(levelname)s %(funcName)s:%(lineno)d %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
os.chmod(logfile, stat.S_IREAD | stat.S_IWRITE)
except IOError as error:
if error.errno == int('13'):
sys.exit('Error while writing to logfile: %s' % strerror)
try:
database = Database('yamls/', includes)
if len(database.issues) == 0:
sys.exit('Empty database. Exiting..')
# stderr to /dev/null
devnull_fd = open(os.devnull, "w")
sys.stderr = devnull_fd
log_to_stderr()
# Starts the asynchronous workers. Amount of workers is the same as cores in server.
# http://docs.python.org/library/multiprocessing.html#multiprocessing.pool.multiprocessing.Pool
logging.debug('Starting workers.')
pool = Pool()
pool.apply_async(Worker, [arguments['--home'], post_process])
# Starts the actual populator daemon to get possible locations, which will be verified by workers.
# http://docs.python.org/library/multiprocessing.html#multiprocessing.Process
p = PopulateScanQueue()
if arguments['-r']:
logging.debug('Scanning recursively from path: %s',
arguments['-r'])
populator = Process(target=p.populate, args=([arguments['-r']], ))
elif arguments['--home']:
logging.debug('Scanning predefined variables: %s',
arguments['--home'])
if __name__ == "__main__":
# load node id's that we have already processed
processed_ids = set()
try:
with open('results/processed_similarity_ids.txt') as file_in:
for line in file_in:
processed_ids.add(int(line))
except:
pass
# process the rest
nodes_to_process = [x for x in nodes_for_comparison if x not in processed_ids]
# parallel generation of results
util.log_to_stderr(util.SUBDEBUG)
n_cores = cpu_count()
chunksize = max(1, len(nodes_to_process) // (2 * (n_cores - 1)))
pool = Pool(processes=(n_cores - 1))
similarity_results = pool.imap(find_missing_edges,
nodes_to_process,
chunksize=chunksize)
# write results to file
with open('results/similarity_results.csv', 'a') as file_out, \
open('results/processed_similarity_ids.txt', 'a') as processed:
for result in similarity_results:
if not result:
continue
def prepare(data):
'''
Try to get current process ready to unpickle process object
'''
old_main_modules.append(sys.modules['__main__'])
if 'name' in data:
process.current_process().name = data['name']
if 'authkey' in data:
process.current_process()._authkey = data['authkey']
if 'log_to_stderr' in data and data['log_to_stderr']:
util.log_to_stderr()
if 'log_level' in data:
util.get_logger().setLevel(data['log_level'])
if 'sys_path' in data:
sys.path = data['sys_path']
if 'sys_argv' in data:
sys.argv = data['sys_argv']
if 'dir' in data:
os.chdir(data['dir'])
if 'orig_dir' in data:
process.ORIGINAL_DIR = data['orig_dir']
if 'main_path' in data:
# XXX (ncoghlan): The following code makes several bogus
# assumptions regarding the relationship between __file__
# and a module's real name. See PEP 302 and issue #10845
main_path = data['main_path']
main_name = os.path.splitext(os.path.basename(main_path))[0]
if main_name == '__init__':
main_name = os.path.basename(os.path.dirname(main_path))
if main_name == '__main__':
main_module = sys.modules['__main__']
main_module.__file__ = main_path
elif main_name != 'ipython':
# Main modules not actually called __main__.py may
# contain additional code that should still be executed
import imp
if main_path is None:
dirs = None
elif os.path.basename(main_path).startswith('__init__.py'):
dirs = [os.path.dirname(os.path.dirname(main_path))]
else:
dirs = [os.path.dirname(main_path)]
assert main_name not in sys.modules, main_name
file, path_name, etc = imp.find_module(main_name, dirs)
try:
# We would like to do "imp.load_module('__main__', ...)"
# here. However, that would cause 'if __name__ ==
# "__main__"' clauses to be executed.
main_module = imp.load_module(
'__parents_main__', file, path_name, etc
)
finally:
if file:
file.close()
sys.modules['__main__'] = main_module
main_module.__name__ = '__main__'
# Try to make the potentially picklable objects in
# sys.modules['__main__'] realize they are in the main
# module -- somewhat ugly.
for obj in list(main_module.__dict__.values()):
try:
if obj.__module__ == '__parents_main__':
obj.__module__ = '__main__'
except Exception:
pass
def prepare(data):
"""
Try to get current process ready to unpickle process object
"""
old_main_modules.append(sys.modules["__main__"])
if "name" in data:
process.current_process().name = data["name"]
if "authkey" in data:
process.current_process()._authkey = data["authkey"]
if "log_to_stderr" in data and data["log_to_stderr"]:
util.log_to_stderr()
if "log_level" in data:
util.get_logger().setLevel(data["log_level"])
if "sys_path" in data:
sys.path = data["sys_path"]
if "sys_argv" in data:
sys.argv = data["sys_argv"]
if "dir" in data:
os.chdir(data["dir"])
if "orig_dir" in data:
process.ORIGINAL_DIR = data["orig_dir"]
if "main_path" in data:
# XXX (ncoghlan): The following code makes several bogus
# assumptions regarding the relationship between __file__
# and a module's real name. See PEP 302 and issue #10845
# The problem is resolved properly in Python 3.4+, as
# described in issue #19946
main_path = data["main_path"]
main_name = os.path.splitext(os.path.basename(main_path))[0]
if main_name == "__init__":
main_name = os.path.basename(os.path.dirname(main_path))
if main_name == "__main__":
# For directory and zipfile execution, we assume an implicit
# "if __name__ == '__main__':" around the module, and don't
# rerun the main module code in spawned processes
main_module = sys.modules["__main__"]
main_module.__file__ = main_path
elif main_name != "ipython":
# Main modules not actually called __main__.py may
# contain additional code that should still be executed
import imp
if main_path is None:
dirs = None
elif os.path.basename(main_path).startswith("__init__.py"):
dirs = [os.path.dirname(os.path.dirname(main_path))]
else:
dirs = [os.path.dirname(main_path)]
assert main_name not in sys.modules, main_name
file, path_name, etc = imp.find_module(main_name, dirs)
try:
# We would like to do "imp.load_module('__main__', ...)"
# here. However, that would cause 'if __name__ ==
# "__main__"' clauses to be executed.
main_module = imp.load_module("__parents_main__", file, path_name, etc)
finally:
if file:
file.close()
sys.modules["__main__"] = main_module
main_module.__name__ = "__main__"
# Try to make the potentially picklable objects in
# sys.modules['__main__'] realize they are in the main
# module -- somewhat ugly.
for obj in main_module.__dict__.values():
try:
if obj.__module__ == "__parents_main__":
obj.__module__ = "__main__"
except Exception:
pass
