def main():
'''
Run code specifed by data received over pipe
'''
global _forking_is_enabled
_Django_old_layout_hack__load()
assert is_forking(sys.argv)
_forking_is_enabled = False
handle = int(sys.argv[-1])
if sys.platform == 'win32':
fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
else:
fd = handle
from_parent = os.fdopen(fd, 'rb')
process.current_process()._inheriting = True
preparation_data = load(from_parent)
prepare(preparation_data)
# Huge hack to make logging before Process.run work.
try:
os.environ["MP_MAIN_FILE"] = sys.modules["__main__"].__file__
except KeyError:
pass
except AttributeError:
pass
loglevel = os.environ.get("_MP_FORK_LOGLEVEL_")
logfile = os.environ.get("_MP_FORK_LOGFILE_") or None
format = os.environ.get("_MP_FORK_LOGFORMAT_")
if loglevel:
from billiard import util
import logging
logger = util.get_logger()
logger.setLevel(int(loglevel))
if not logger.handlers:
logger._rudimentary_setup = True
logfile = logfile or sys.__stderr__
if hasattr(logfile, "write"):
handler = logging.StreamHandler(logfile)
else:
handler = logging.FileHandler(logfile)
formatter = logging.Formatter(
format or util.DEFAULT_LOGGING_FORMAT,
)
handler.setFormatter(formatter)
logger.addHandler(handler)
self = load(from_parent)
process.current_process()._inheriting = False
from_parent.close()
exitcode = self._bootstrap()
exit(exitcode)
def get_preparation_data(name):
'''
Return info about parent needed by child to unpickle process object
'''
from billiard.util import _logger, _log_to_stderr
d = dict(
name=name,
sys_path=sys.path,
sys_argv=sys.argv,
log_to_stderr=_log_to_stderr,
orig_dir=process.ORIGINAL_DIR,
authkey=process.current_process().authkey,
)
if _logger is not None:
d['log_level'] = _logger.getEffectiveLevel()
if not WINEXE and not WINSERVICE:
main_path = getattr(sys.modules['__main__'], '__file__', None)
if not main_path and sys.argv[0] not in ('', '-c'):
main_path = sys.argv[0]
if main_path is not None:
if (not os.path.isabs(main_path) and
process.ORIGINAL_DIR is not None):
main_path = os.path.join(process.ORIGINAL_DIR, main_path)
d['main_path'] = os.path.normpath(main_path)
return d
def test_worker_term_hard_handler_only_stop_MainProcess(self):
try:
import _multiprocessing # noqa
except ImportError:
raise SkipTest('only relevant for multiprocessing')
process = current_process()
name, process.name = process.name, 'OtherProcess'
try:
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 3
worker = self._Worker()
handlers = self.psig(
cd.install_worker_term_hard_handler, worker)
try:
handlers['SIGQUIT']('SIGQUIT', object())
self.assertTrue(state.should_terminate)
finally:
state.should_terminate = None
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 1
worker = self._Worker()
handlers = self.psig(
cd.install_worker_term_hard_handler, worker)
try:
with self.assertRaises(WorkerTerminate):
handlers['SIGQUIT']('SIGQUIT', object())
finally:
state.should_terminate = None
finally:
process.name = name
def current_process():
try:
from billiard import process
except ImportError:
pass
else:
return process.current_process()
def test_worker_int_handler_only_stop_MainProcess(self):
try:
import _multiprocessing # noqa
except ImportError:
raise SkipTest("only relevant for multiprocessing")
process = current_process()
name, process.name = process.name, "OtherProcess"
with patch("celery.apps.worker.active_thread_count") as c:
c.return_value = 3
try:
worker = self._Worker()
handlers = self.psig(cd.install_worker_int_handler, worker)
handlers["SIGINT"]("SIGINT", object())
self.assertTrue(state.should_stop)
finally:
process.name = name
state.should_stop = None
with patch("celery.apps.worker.active_thread_count") as c:
c.return_value = 1
try:
worker = self._Worker()
handlers = self.psig(cd.install_worker_int_handler, worker)
with self.assertRaises(WorkerShutdown):
handlers["SIGINT"]("SIGINT", object())
finally:
process.name = name
state.should_stop = None
def test_worker_term_hard_handler_only_stop_MainProcess(self):
process = current_process()
name, process.name = process.name, 'OtherProcess'
try:
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 3
worker = self._Worker()
handlers = self.psig(
cd.install_worker_term_hard_handler, worker)
try:
handlers['SIGQUIT']('SIGQUIT', object())
assert state.should_terminate
finally:
state.should_terminate = None
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 1
worker = self._Worker()
handlers = self.psig(
cd.install_worker_term_hard_handler, worker)
try:
with pytest.raises(WorkerTerminate):
handlers['SIGQUIT']('SIGQUIT', object())
finally:
state.should_terminate = None
finally:
process.name = name
def current_process():
try:
from billiard import process
except ImportError: # pragma: no cover
pass
else:
return process.current_process()
def get_command_line():
'''
Returns prefix of command line used for spawning a child process
'''
if process.current_process()._identity == () and is_forking(sys.argv):
raise RuntimeError('''
Attempt to start a new process before the current process
has finished its bootstrapping phase.
This probably means that have forgotten to use the proper
idiom in the main module:
if __name__ == '__main__':
freeze_support()
...
The "freeze_support()" line can be omitted if the program
is not going to be frozen to produce a Windows executable.''')
if getattr(sys, 'frozen', False):
return [sys.executable, '--billiard-fork']
else:
prog = 'from billiard.forking import main; main()'
os.environ['PYTHONPATH'] = ':'.join(sys.path)
return [_python_exe, '-c', prog, '--billiard-fork']
def get_command_line():
"""
Returns prefix of command line used for spawning a child process
"""
if process.current_process()._identity == () and is_forking(sys.argv):
raise RuntimeError(
"""
Attempt to start a new process before the current process
has finished its bootstrapping phase.
This probably means that have forgotten to use the proper
idiom in the main module:
if __name__ == '__main__':
freeze_support()
...
The "freeze_support()" line can be omitted if the program
is not going to be frozen to produce a Windows executable."""
)
if getattr(sys, "frozen", False):
return [sys.executable, "--billiard-fork"]
else:
prog = "from billiard.forking import main; main()"
return [_python_exe, "-c", prog, "--billiard-fork"]
def set_mp_process_title(progname, info=None, hostname=None): # noqa
"""Set the :command:`ps` name from the current process name.
Only works if :pypi:`setproctitle` is installed.
"""
if hostname:
progname = '{0}: {1}'.format(progname, hostname)
name = current_process().name if current_process else 'MainProcess'
return set_process_title('{0}:{1}'.format(progname, name), info=info)
def set_mp_process_title(progname, info=None, hostname=None): # noqa
"""Set the ps name using the multiprocessing process name.
Only works if :mod:`setproctitle` is installed.
"""
if hostname:
progname = '{0}: {1}'.format(progname, hostname)
name = current_process().name if current_process else 'MainProcess'
return set_process_title('{0}:{1}'.format(progname, name), info=info)
def set_mp_process_title(progname, info=None, hostname=None): # noqa
"""Set the :command:`ps` name using the :mod:`multiprocessing`
process name.
Only works if :pypi:`setproctitle` is installed.
"""
if hostname:
progname = "{0}: {1}".format(progname, hostname)
name = current_process().name if current_process else "MainProcess"
return set_process_title("{0}:{1}".format(progname, name), info=info)
def _handle_request(*args):
with in_sighandler():
from celery.worker import state
if current_process()._name == "MainProcess":
if callback:
callback(worker)
safe_say("worker: {0} shutdown (MainProcess)".format(how))
if active_thread_count() > 1:
setattr(state, {"Warm": "should_stop", "Cold": "should_terminate"}[how], exitcode)
else:
raise exc(exitcode)
def _handle_request(*args):
with in_sighandler():
from celery.worker import state
if current_process()._name == 'MainProcess':
if callback:
callback(worker)
safe_say('worker: {0} shutdown (MainProcess)'.format(how))
if active_thread_count() > 1:
setattr(state, {'Warm': 'should_stop',
'Cold': 'should_terminate'}[how], exitcode)
else:
raise exc(exitcode)
def _app_or_default_trace(app=None): # pragma: no cover
from traceback import print_stack
try:
from billiard.process import current_process
except ImportError:
current_process = None
if app is None:
if getattr(_state._tls, 'current_app', None):
print('-- RETURNING TO CURRENT APP --') # noqa+
print_stack()
return _state._tls.current_app
if not current_process or current_process()._name == 'MainProcess':
raise Exception('DEFAULT APP')
print('-- RETURNING TO DEFAULT APP --') # noqa+
print_stack()
return _state.default_app
return app
def get_preparation_data(name):
'''
Return info about parent needed by child to unpickle process object
'''
from billiard.util import _logger, _log_to_stderr
d = dict(
name=name,
sys_path=sys.path,
sys_argv=sys.argv,
log_to_stderr=_log_to_stderr,
orig_dir=process.ORIGINAL_DIR,
authkey=process.current_process().authkey,
)
if _logger is not None:
d['log_level'] = _logger.getEffectiveLevel()
if not WINEXE and not WINSERVICE:
main_path = getattr(sys.modules['__main__'], '__file__', None)
if not main_path and sys.argv[0] not in ('', '-c'):
main_path = sys.argv[0]
if main_path is not None:
if (not os.path.isabs(main_path) and
process.ORIGINAL_DIR is not None):
main_path = os.path.join(process.ORIGINAL_DIR, main_path)
d['main_path'] = os.path.normpath(main_path)
return d
#
# Make (Pipe)Connection picklable
#
def reduce_connection(conn):
if not Popen.thread_is_spawning():
raise RuntimeError(
'By default %s objects can only be shared between processes\n'
'using inheritance' % type(conn).__name__
)
return type(conn), (Popen.duplicate_for_child(conn.fileno()),
conn.readable, conn.writable)
ForkingPickler.register(Connection, reduce_connection)
ForkingPickler.register(PipeConnection, reduce_connection)
def test_worker_term_handler_only_stop_MainProcess(self):
process = current_process()
name, process.name = process.name, 'OtherProcess'
try:
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 3
worker = self._Worker()
handlers = self.psig(cd.install_worker_term_handler, worker)
handlers['SIGTERM']('SIGTERM', object())
self.assertEqual(state.should_stop, EX_OK)
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 1
worker = self._Worker()
handlers = self.psig(cd.install_worker_term_handler, worker)
with self.assertRaises(WorkerShutdown):
handlers['SIGTERM']('SIGTERM', object())
finally:
process.name = name
state.should_stop = None
def get_avail_port(self, host, port, search_limit=100, skew=+0):
try:
_, skew = current_process().name.split('-')
skew = int(skew)
except ValueError:
pass
this_port = None
for i in range(search_limit):
_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
this_port = port + skew + i
try:
_sock.bind((host, this_port))
except socket.error as exc:
if exc.errno in [errno.EADDRINUSE, errno.EINVAL]:
continue
raise
else:
return _sock, this_port
else:
raise Exception(NO_AVAILABLE_PORT.format(self=self))
def test_worker_int_handler_only_stop_MainProcess(self):
process = current_process()
name, process.name = process.name, 'OtherProcess'
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 3
try:
worker = self._Worker()
handlers = self.psig(cd.install_worker_int_handler, worker)
handlers['SIGINT']('SIGINT', object())
assert state.should_stop
finally:
process.name = name
state.should_stop = None
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 1
try:
worker = self._Worker()
handlers = self.psig(cd.install_worker_int_handler, worker)
with pytest.raises(WorkerShutdown):
handlers['SIGINT']('SIGINT', object())
finally:
process.name = name
state.should_stop = None
import atexit
from billiard.process import current_process
from celery.five import monotonic
from celery.utils.debug import memdump, sample_mem
all_count = 0
bench_first = None
bench_start = None
bench_last = None
bench_every = C_BENCH_EVERY
bench_sample = []
__reserved = task_reserved
__ready = task_ready
if current_process()._name == "MainProcess":
@atexit.register
def on_shutdown():
if bench_first is not None and bench_last is not None:
print("- Time spent in benchmark: {0!r}".format(bench_last -
bench_first))
print("- Avg: {0}".format(
sum(bench_sample) / len(bench_sample)))
memdump()
def task_reserved(request): # noqa
"""Called when a task is reserved by the worker."""
global bench_start
global bench_first
now = None
def prepare(data):
'''
Try to get current process ready to unpickle process object
'''
old_main_modules.append(sys.modules['__main__'])
if 'name' in data:
process.current_process().name = data['name']
if 'authkey' in data:
process.current_process()._authkey = data['authkey']
if 'log_to_stderr' in data and data['log_to_stderr']:
util.log_to_stderr()
if 'log_level' in data:
util.get_logger().setLevel(data['log_level'])
if 'sys_path' in data:
sys.path = data['sys_path']
if 'sys_argv' in data:
sys.argv = data['sys_argv']
if 'dir' in data:
os.chdir(data['dir'])
if 'orig_dir' in data:
process.ORIGINAL_DIR = data['orig_dir']
if 'main_path' in data:
main_path = data['main_path']
main_name = os.path.splitext(os.path.basename(main_path))[0]
if main_name == '__init__':
main_name = os.path.basename(os.path.dirname(main_path))
if main_name == '__main__':
main_module = sys.modules['__main__']
main_module.__file__ = main_path
elif main_name != 'ipython':
# Main modules not actually called __main__.py may
# contain additional code that should still be executed
import imp
if main_path is None:
dirs = None
elif os.path.basename(main_path).startswith('__init__.py'):
dirs = [os.path.dirname(os.path.dirname(main_path))]
else:
dirs = [os.path.dirname(main_path)]
assert main_name not in sys.modules, main_name
file, path_name, etc = imp.find_module(main_name, dirs)
try:
# We would like to do "imp.load_module('__main__', ...)"
# here. However, that would cause 'if __name__ ==
# "__main__"' clauses to be executed.
main_module = imp.load_module(
'__parents_main__', file, path_name, etc
)
finally:
if file:
file.close()
sys.modules['__main__'] = main_module
main_module.__name__ = '__main__'
# Try to make the potentially picklable objects in
# sys.modules['__main__'] realize they are in the main
# module -- somewhat ugly.
for obj in list(main_module.__dict__.values()):
try:
if obj.__module__ == '__parents_main__':
obj.__module__ = '__main__'
except Exception:
pass
import atexit
from billiard.process import current_process
from celery.five import monotonic
from celery.utils.debug import memdump, sample_mem
all_count = 0
bench_first = None
bench_start = None
bench_last = None
bench_every = C_BENCH_EVERY
bench_sample = []
__reserved = task_reserved
__ready = task_ready
if current_process()._name == 'MainProcess':
@atexit.register
def on_shutdown():
if bench_first is not None and bench_last is not None:
print('- Time spent in benchmark: {0!r}'.format(
bench_last - bench_first))
print('- Avg: {0}'.format(
sum(bench_sample) / len(bench_sample)))
memdump()
def task_reserved(request): # noqa
global bench_start
global bench_first
now = None
if bench_start is None:
bench_start = now = monotonic()
def prepare(data):
'''
Try to get current process ready to unpickle process object
'''
old_main_modules.append(sys.modules['__main__'])
if 'name' in data:
process.current_process().name = data['name']
if 'authkey' in data:
process.current_process()._authkey = data['authkey']
if 'log_to_stderr' in data and data['log_to_stderr']:
util.log_to_stderr()
if 'log_level' in data:
util.get_logger().setLevel(data['log_level'])
if 'sys_path' in data:
sys.path = data['sys_path']
if 'sys_argv' in data:
sys.argv = data['sys_argv']
if 'dir' in data:
os.chdir(data['dir'])
if 'orig_dir' in data:
process.ORIGINAL_DIR = data['orig_dir']
if 'main_path' in data:
main_path = data['main_path']
main_name = os.path.splitext(os.path.basename(main_path))[0]
if main_name == '__init__':
main_name = os.path.basename(os.path.dirname(main_path))
if main_name == '__main__':
main_module = sys.modules['__main__']
main_module.__file__ = main_path
elif main_name != 'ipython':
# Main modules not actually called __main__.py may
# contain additional code that should still be executed
import imp
if main_path is None:
dirs = None
elif os.path.basename(main_path).startswith('__init__.py'):
dirs = [os.path.dirname(os.path.dirname(main_path))]
else:
dirs = [os.path.dirname(main_path)]
assert main_name not in sys.modules, main_name
file, path_name, etc = imp.find_module(main_name, dirs)
try:
# We would like to do "imp.load_module('__main__', ...)"
# here. However, that would cause 'if __name__ ==
# "__main__"' clauses to be executed.
main_module = imp.load_module('__parents_main__', file,
path_name, etc)
finally:
if file:
file.close()
sys.modules['__main__'] = main_module
main_module.__name__ = '__main__'
# Try to make the potentially picklable objects in
# sys.modules['__main__'] realize they are in the main
# module -- somewhat ugly.
for obj in list(main_module.__dict__.values()):
try:
if obj.__module__ == '__parents_main__':
obj.__module__ = '__main__'
except Exception:
pass
import atexit
from billiard.process import current_process
from celery.five import monotonic
from celery.utils.debug import memdump, sample_mem
all_count = 0
bench_first = None
bench_start = None
bench_last = None
bench_every = C_BENCH_EVERY
bench_sample = []
__reserved = task_reserved
__ready = task_ready
if current_process()._name == "MainProcess":
@atexit.register
def on_shutdown():
if bench_first is not None and bench_last is not None:
print("- Time spent in benchmark: {0!r}".format(bench_last - bench_first))
print("- Avg: {0}".format(sum(bench_sample) / len(bench_sample)))
memdump()
def task_reserved(request): # noqa
"""Called when a task is reserved by the worker."""
global bench_start
global bench_first
now = None
if bench_start is None:
bench_start = now = monotonic()
import atexit
from billiard.process import current_process
from celery.five import monotonic
from celery.utils.debug import memdump, sample_mem
all_count = 0
bench_first = None
bench_start = None
bench_last = None
bench_every = C_BENCH_EVERY
bench_sample = []
__reserved = task_reserved
__ready = task_ready
if current_process()._name == 'MainProcess':
@atexit.register
def on_shutdown():
if bench_first is not None and bench_last is not None:
print('- Time spent in benchmark: {0!r}'.format(bench_last -
bench_first))
print('- Avg: {0}'.format(
sum(bench_sample) / len(bench_sample)))
memdump()
def task_reserved(request): # noqa
global bench_start
global bench_first
now = None
if bench_start is None:
