def getPreparationData(name):
"""
Return info about parent needed by child to unpickle process object
"""
from processing.logger import _logger
if _logger is not None:
log_args = (_logger.getEffectiveLevel(),) + _logger._extra_args
else:
log_args = None
if sys.argv[0] not in ("", "-c") and not WINEXE:
mainpath = getattr(sys.modules["__main__"], "__file__", None)
if mainpath is not None and not os.path.isabs(mainpath):
# we will assume os.chdir() was not used between program
# start up and the first import of processing
mainpath = os.path.join(processing.ORIGINAL_DIR, mainpath)
else:
mainpath = None
return [
name,
mainpath,
sys.path,
sys.argv,
processing.currentProcess().getAuthKey(),
None,
processing.ORIGINAL_DIR,
log_args,
]
def getCommandLine():
'''
Returns prefix of command line used for spawning a child process
'''
if processing.currentProcess()._identity==() and isForking(sys.argv):
raise RuntimeError, '''
Attempt to start a new process before the current process
has finished its bootstrapping phase.
This probably means that you are on Windows and you have
forgotten to use the proper idiom in the main module:
if __name__ == '__main__':
freezeSupport()
...
The "freezeSupport()" line can be omitted if the program
is not going to be frozen to produce a Windows executable.'''
prog = 'from processing.forking import main; main()'
if getattr(sys, 'frozen', False):
return [sys.executable, '--processing-fork']
elif sys.executable.lower().endswith('pythonservice.exe'):
exe = os.path.join(os.path.dirname(os.__file__),'..','python.exe')
return [exe, '-c', prog, '--processing-fork']
else:
return [sys.executable, '-c', prog, '--processing-fork']
def getCommandLine():
"""
Returns prefix of command line used for spawning a child process
"""
if processing.currentProcess()._identity == () and isForking(sys.argv):
raise RuntimeError, """
Attempt to start a new process before the current process
has finished its bootstrapping phase.
This probably means that you are on Windows and you have
forgotten to use the proper idiom in the main module:
if __name__ == '__main__':
freezeSupport()
...
The "freezeSupport()" line can be omitted if the program
is not going to be frozen to produce a Windows executable."""
prog = "from processing.forking import main; main()"
if getattr(sys, "frozen", False):
return [sys.executable, "--processing-fork"]
elif sys.executable.lower().endswith("pythonservice.exe"):
exe = os.path.join(os.path.dirname(os.__file__), "..", "python.exe")
return [exe, "-c", prog, "--processing-fork"]
else:
return [sys.executable, "-c", prog, "--processing-fork"]
def value_func(running, mutex):
random.seed()
time.sleep(random.random() * 4)
mutex.acquire()
print '\n\t\t\t' + str(processing.currentProcess()) + ' has finished'
running.value -= 1
mutex.release()
def value_func(running, mutex):
random.seed()
time.sleep(random.random()*4)
mutex.acquire()
print '\n\t\t\t' + str(processing.currentProcess()) + ' has finished'
running.value -= 1
mutex.release()
def __init__(self, address=None, family=None, backlog=1,
authenticate=False, authkey=None):
'''
`address`
The address to be used by the bound socket
or named pipe of `self`.
`family`
The type of the socket or named pipe to use.
This can be one of the strings 'AF_INET' (for a TCP
socket), 'AF_UNIX' (for a Unix domain socket) or 'AF_PIPE'
(for a Windows named pipe). Of these only the first is
guaranteed to be available.
If `family` is None than the family is inferred by the
format of `address`. If `address` is unspecified then
a default is chosen which is dependent on the platform.
This default is the family which is assumed to be the
fastest available.
`backlog`
If the `self` uses a socket then this is passed to the
`listen()` method of the socket once it has been bound.
`authenticate`
If this is true then digest authentication is used even if
`authkey` is` None`.
`authkey`
If `authkey` is a string then it will be used as the
authentication key; otherwise it must be `None`.
If `authkey` is `None` and `authenticate` is true then
`currentProcess.getAuthKey()` is used as the authentication
key.
If `authkey` is `None` and `authentication` is false then
no authentication is done.
'''
family = family or (address and addressType(address)) \
or default_family
address = address or arbitraryAddress(family)
if family == 'AF_PIPE':
self._listener = PipeListener(address, backlog)
else:
self._listener = SocketListener(address, family, backlog)
if authenticate and authkey is None:
authkey = currentProcess().getAuthKey()
elif authenticate:
assert type(authkey) is str
self._authkey = authkey
def main():
"""
Run code specifed by data received over pipe
"""
assert isForking(sys.argv)
handle = int(sys.argv[-1])
fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
from_parent = os.fdopen(fd, "rb")
processing.currentProcess()._inheriting = True
preparation_data = load(from_parent)
prepare(*preparation_data)
self = load(from_parent)
processing.currentProcess()._inheriting = False
from_parent.close()
exitcode = self._bootstrap()
win32.ExitProcess(exitcode)
def main():
'''
Run code specifed by data received over pipe
'''
assert isForking(sys.argv)
handle = int(sys.argv[-1])
fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
from_parent = os.fdopen(fd, 'rb')
processing.currentProcess()._inheriting = True
preparation_data = load(from_parent)
prepare(*preparation_data)
self = load(from_parent)
processing.currentProcess()._inheriting = False
from_parent.close()
exitcode = self._bootstrap()
win32.ExitProcess(exitcode)
def _feed(buffer, notempty, send, writelock, close):
debug('starting thread to feed data to pipe')
nacquire = notempty.acquire
nrelease = notempty.release
nwait = notempty.wait
bpopleft = buffer.popleft
sentinel = _sentinel
if sys.platform != 'win32':
wacquire = writelock.acquire
wrelease = writelock.release
else:
wacquire = None
try:
while 1:
nacquire()
try:
if not buffer:
nwait()
finally:
nrelease()
try:
while 1:
obj = bpopleft()
if obj is sentinel:
debug('feeder thread got sentinel -- exiting')
close()
return
if wacquire is None:
send(obj)
else:
wacquire()
try:
send(obj)
finally:
wrelease()
except IndexError:
pass
except Exception, e:
# Since this runs in a daemon thread the objects it uses
# may be become unusable while the process is cleaning up.
# We ignore errors which happen after the process has
# started to cleanup.
if currentProcess()._exiting:
subWarning('error in queue thread: %s', e)
else:
raise
def _feed(buffer, notempty, send, writelock, close):
debug('starting thread to feed data to pipe')
nacquire = notempty.acquire
nrelease = notempty.release
nwait = notempty.wait
bpopleft = buffer.popleft
sentinel = _sentinel
if sys.platform != 'win32':
wacquire = writelock.acquire
wrelease = writelock.release
else:
wacquire = None
try:
while 1:
nacquire()
try:
if not buffer:
nwait()
finally:
nrelease()
try:
while 1:
obj = bpopleft()
if obj is sentinel:
debug('feeder thread got sentinel -- exiting')
close()
return
if wacquire is None:
send(obj)
else:
wacquire()
try:
send(obj)
finally:
wrelease()
except IndexError:
pass
except Exception, e:
# Since this runs in a daemon thread the objects it uses
# may be become unusable while the process is cleaning up.
# We ignore errors which happen after the process has
# started to cleanup.
if currentProcess()._exiting:
subWarning('error in queue thread: %s', e)
else:
raise
def _serve():
while 1:
try:
conn = _listener.accept()
handle_wanted, destination_pid = conn.recv()
_cache.remove(handle_wanted)
sendHandle(conn, handle_wanted, destination_pid)
closeHandle(handle_wanted)
conn.close()
except (SystemExit, KeyboardInterrupt):
raise
except:
if not processing.currentProcess()._exiting:
import traceback
subWarning('thread for sharing handles raised exception :\n' +
'-' * 79 + '\n' + traceback.format_exc() + '-' * 79)
def semaphore_func(sema, mutex, running):
sema.acquire()
mutex.acquire()
running.value += 1
print running.value, 'tasks are running'
mutex.release()
random.seed()
time.sleep(random.random() * 2)
mutex.acquire()
running.value -= 1
print '%s has finished' % processing.currentProcess()
mutex.release()
sema.release()
def semaphore_func(sema, mutex, running):
sema.acquire()
mutex.acquire()
running.value += 1
print running.value, 'tasks are running'
mutex.release()
random.seed()
time.sleep(random.random()*2)
mutex.acquire()
running.value -= 1
print '%s has finished' % processing.currentProcess()
mutex.release()
sema.release()
def _serve():
while 1:
try:
conn = _listener.accept()
handle_wanted, destination_pid = conn.recv()
_cache.remove(handle_wanted)
sendHandle(conn, handle_wanted, destination_pid)
closeHandle(handle_wanted)
conn.close()
except (SystemExit, KeyboardInterrupt):
raise
except:
if not processing.currentProcess()._exiting:
import traceback
subWarning(
'thread for sharing handles raised exception :\n' +
'-'*79 + '\n' + traceback.format_exc() + '-'*79
)
def Client(address, family=None, authenticate=False, authkey=None):
'''
Returns a connection to the address of a `Listener`
'''
family = family or addressType(address)
if family == 'AF_PIPE':
c = PipeClient(address)
else:
c = SocketClient(address)
if authenticate and authkey is None:
authkey = currentProcess().getAuthKey()
elif authenticate:
assert type(authkey) is str
if authkey is not None:
answerChallenge(c, authkey)
deliverChallenge(c, authkey)
return c
def getPreparationData(name):
'''
Return info about parent needed by child to unpickle process object
'''
from processing.logger import _logger
if _logger is not None:
log_args = (_logger.getEffectiveLevel(),) + _logger._extra_args
else:
log_args = None
if sys.argv[0] not in ('', '-c') and not WINEXE:
mainpath = getattr(sys.modules['__main__'], '__file__', None)
if mainpath is not None and not os.path.isabs(mainpath):
# we will assume os.chdir() was not used between program
# start up and the first import of processing
mainpath = os.path.join(processing.ORIGINAL_DIR, mainpath)
else:
mainpath = None
return [name, mainpath, sys.path, sys.argv,
processing.currentProcess().getAuthKey(),
None, processing.ORIGINAL_DIR, log_args]
def calculate(func, args):
result = func(*args)
return '%s says that %s%s = %s' % \
(currentProcess().getName(), func.__name__, args, result)
def note(format, *args):
sys.stderr.write('[%s]\t%s\n' %
(currentProcess().getName(), format % args))
def prepare(name, mainpath, sys_path, sys_argv, authkey,
cur_dir, orig_dir, log_args):
'''
Try to get this process ready to unpickle process object
'''
global original_main_module
original_main_module = sys.modules['__main__']
processing.currentProcess().setName(name)
processing.currentProcess().setAuthKey(authkey)
if log_args is not None:
from processing.logger import enableLogging
enableLogging(*log_args)
if orig_dir is not None:
processing.ORIGINAL_DIR = orig_dir
if cur_dir is not None:
try:
os.chdir(cur_dir)
except OSError:
raise
if sys_path is not None:
sys.path = sys_path
if mainpath is not None:
mainname = splitext(basename(mainpath))[0]
if mainname == '__init__':
mainname = basename(dirname(mainpath))
if not mainpath.lower().endswith('.exe') and mainname != 'ipython':
if mainpath is None:
dirs = None
elif basename(mainpath).startswith('__init__.py'):
dirs = [dirname(dirname(mainpath))]
else:
dirs = [dirname(mainpath)]
assert mainname not in sys.modules, mainname
file, pathname, etc = imp.find_module(mainname, dirs)
try:
# We would like to do "imp.load_module('__main__', ...)"
# here. However, that would cause 'if __name__ ==
# "__main__"' clauses to be executed.
main_module = imp.load_module(
'__parents_main__', file, pathname, etc
)
finally:
if file:
file.close()
sys.modules['__main__'] = main_module
main_module.__name__ = '__main__'
# XXX Try to make the potentially picklable objects in
# sys.modules['__main__'] realize they are in the main
# module -- ugly
for obj in main_module.__dict__.values():
try:
if obj.__module__ == '__parents_main__':
obj.__module__ = '__main__'
except (KeyboardInterrupt, SystemExit):
raise
except:
pass
if sys_argv is not None: # this needs to come last
sys.argv = sys_argv
def note(format, *args):
sys.stderr.write('[%s]\t%s\n' % (currentProcess().getName(), format%args))
def event_func(event):
print '\t%r is waiting' % processing.currentProcess()
event.wait()
print '\t%r has woken up' % processing.currentProcess()
def prepare(name, mainpath, sys_path, sys_argv, authkey, cur_dir, orig_dir, log_args):
"""
Try to get this process ready to unpickle process object
"""
global original_main_module
original_main_module = sys.modules["__main__"]
processing.currentProcess().setName(name)
processing.currentProcess().setAuthKey(authkey)
if log_args is not None:
from processing.logger import enableLogging
enableLogging(*log_args)
if orig_dir is not None:
processing.ORIGINAL_DIR = orig_dir
if cur_dir is not None:
try:
os.chdir(cur_dir)
except OSError:
raise
if sys_path is not None:
sys.path = sys_path
if mainpath is not None:
mainname = splitext(basename(mainpath))[0]
if mainname == "__init__":
mainname = basename(dirname(mainpath))
if not mainpath.lower().endswith(".exe") and mainname != "ipython":
if mainpath is None:
dirs = None
elif basename(mainpath).startswith("__init__.py"):
dirs = [dirname(dirname(mainpath))]
else:
dirs = [dirname(mainpath)]
assert mainname not in sys.modules, mainname
file, pathname, etc = imp.find_module(mainname, dirs)
try:
# We would like to do "imp.load_module('__main__', ...)"
# here. However, that would cause 'if __name__ ==
# "__main__"' clauses to be executed.
main_module = imp.load_module("__parents_main__", file, pathname, etc)
finally:
if file:
file.close()
sys.modules["__main__"] = main_module
main_module.__name__ = "__main__"
# XXX Try to make the potentially picklable objects in
# sys.modules['__main__'] realize they are in the main
# module -- ugly
for obj in main_module.__dict__.values():
try:
if obj.__module__ == "__parents_main__":
obj.__module__ = "__main__"
except (KeyboardInterrupt, SystemExit):
raise
except:
pass
if sys_argv is not None: # this needs to come last
sys.argv = sys_argv
def event_func(event):
print '\t%r is waiting' % processing.currentProcess()
event.wait()
print '\t%r has woken up' % processing.currentProcess()
def calculate(func, args):
result = func(*args)
return '%s says that %s%s = %s' % \
(currentProcess().getName(), func.__name__, args, result)
