"""map(function, sequence[, sequence, ...]) -> list

Like the builtin map() function, but splits the workload across a

pool of processes whenever possible.

>>> map(None, [1,2,3])

[1, 2, 3]

>>> map(None, [1,2,3], [5,6])

[(1, 5), (2, 6), (3, None)]

"""

# IPC stuff

structformat = 'H'

structlen = struct.calcsize(structformat)

def sendmessage(myend, message):

"""Send a pickled message across a pipe"""

outobj = cPickle.dumps(message)

os.write(myend, struct.pack(structformat, len(outobj)) + outobj)

def recvmessage(myend):

"""Receive a pickled message from a pipe"""

length = struct.unpack(structformat, (os.read(myend, structlen)))[0]

return cPickle.loads(os.read(myend, length))

try:

maxchildren = function.parallel_maxchildren

except AttributeError:

return __builtin__.map(function, *sequence)

# Handle map()'s multi-sequence semantics

if len(sequence) == 1:

if function is None:

return list(sequence[0])

arglist = zip(sequence[0])

else:

arglist = __builtin__.map(None, *sequence)

if function is None:

return arglist

argindex = 0

finished = 0

outlist = [None] * len(arglist)

# Spawn the worker children. Don't create more than the number of

# values we'll be processing.

fromchild, toparent = os.pipe()

children = []

for childnum in range(min(maxchildren, len(arglist))):

fromparent, tochild = os.pipe()

pid = os.fork()

# Parent?

if pid:

# Do some housekeeping and give the child its first assignment

children.append({'pid' : pid,

'fromparent': fromparent,

'tochild' : tochild,

})

sendmessage(tochild, (argindex, arglist[argindex]))

argindex += 1

# Child?

else:

# Since children can't really tell when they've been

# orphaned, set a timeout so that they die if they don't

# hear from the parent in a timely manner.

def timeouthandler(signum, frame):

"""Get out cleanly"""

sys.exit()

oldsignal = signal.signal(signal.SIGALRM, timeouthandler)

# Keep processing values until the parent kills you

while True:

try:

# Wait one second before quitting. Children

# should generally hear from their parent almost

# instantly.

signal.alarm(1)

message = recvmessage(fromparent)

signal.alarm(0)

if message is None:

sys.exit()

index, value = message

sendmessage(toparent, (childnum, index, function(*value)))

except Exception, excvalue:

sendmessage(toparent, (childnum, index, excvalue))

finally:

signal.signal(signal.SIGALRM, oldsignal)

# Keep accepting values back from the children until they've all

# come back

while finished < len(arglist):

returnchild, returnindex, value = recvmessage(fromchild)

if isinstance(value, Exception):

raise value

outlist[returnindex] = value

finished += 1

# If there are still values left to process, hand one back out

# to the child that just finished

if argindex < len(arglist):

sendmessage(children[returnchild]['tochild'],

(argindex, arglist[argindex]))

argindex += 1

# Kill the child processes

[sendmessage(child['tochild'], None) for child in children]

[os.wait() for child in children]

"""Mark a function as eligible for parallelized execution. The

function will run across a number of processes equal to

maxchildren, perproc times the number of processors installed on

the system, or the number of times the function needs to be run to

process all data passed to it - whichever is least."""

if perproc is not None:

processors = 4 # hand-waving

maxchildren = min(maxchildren, perproc * processors)

def decorate(f):

"""Set the parallel_maxchildren attribute to the value

appropriate for this function"""

setattr(f, 'parallel_maxchildren', maxchildren)

return f

"""Make pylint happy"""

for i in range(1000000):

x = x + i