def play(self):
'''round-robin'''
self.open_start_urls()
loops_without_progress = 0
while True:
import time
start = time.time()
if len(self.bots) == 0:
return
# bots got stuck if there's 2 wait pages in a row
if loops_without_progress > 10:
raise AssertionError('Bots got stuck')
results = Chan(buflen=len(self.bots))
threads = []
for bot in self.bots.values():
if bot.on_wait_page():
pass
else:
thread = SubmitThread(bot, results)
threads.append(thread)
thread.start()
for thread in threads:
bot, status = results.get()
if isinstance(status, Exception):
raise status
elif status == 'finished':
del self.bots[bot.participant.id]
else:
bot.submit(status)
results.close()
if not threads:
loops_without_progress += 1
class Proc(object):
"""
One of the processes that runs the copy of app
"""
def __init__(self, app, run_args):
self.marked_for_reload = 0
self.proc_lock = threading.Lock()
self.hosts = []
self.print_tracebacks = False
self.new_host_check_interval = -1
self.app = app
self.stopping = 0
self.wsgis_active = 0
self.chan = Chan()
self.syntax_error = None
try:
self.chan.run_file(run_args)
except Exception, e:
print "Can't run file '%s' (probably syntax errors), "\
"and possibly can't detect it's hosts" % self.app.fn
self.syntax_error = e.args[0]#traceback.format_exc()
self.app.on_proc_restarted(self)
self.force_hosts_check()
def close(self):
errc = Chan()
self.quit.put(errc)
result = errc.get()
self.thread.join(0.2)
assert not self.thread.is_alive()
return result
def __init__(self, subscriptions):
self.subscriptions = subscriptions
self.updates_chan = Chan()
self.quit = Chan()
self.thread = threading.Thread(name="Merged", target=self._run)
self.thread.start()
def test_select_and_closed(self):
a, b, c = [Chan() for _ in range(3)]
out = Chan()
go(sayset, a, [0, 1, 2], delay=0.01, __name='sayset1')
go(sayset, b, [3, 4, 5], delay=0.01, __name='sayset2')
go(sayset, c, [6, 7, 8], delay=0.01, __name='sayset2')
def fanin_until_closed(inchans, outchan):
inchans = inchans[:]
while inchans:
try:
_, val = select(inchans, [])
out.put(val)
except ChanClosed as ex:
inchans.remove(ex.which)
out.close()
go(fanin_until_closed, [a, b, c], out, __name='fanin')
into = []
acc = go(accumulator, out, into)
acc.join(10)
self.assertFalse(acc.is_alive())
results = set(into)
self.assertEqual(len(results), 9)
self.assertEqual(results, set(range(9)))
class DiscoTimer(object):
def __init__(self, base=1000, jitter=500):
self.trip = False
self._stop = threading.Event()
self.chan = Chan()
self._thread = threading.Thread(name='LightTimer', target=self.run)
self._thread.daemon = True
self._thread.start()
self.base = base
self.jitter = jitter
random.seed()
def run(self):
while not self._stop.is_set():
random_lights = []
for l in lights.ALL:
maybe = random.randint(0, 1)
if maybe == 1:
random_lights.append(l)
self.chan.put(random_lights)
d = random.randint(0, 1)
ti = self.base
ji = random.randint(0, self.jitter)
if d == 0:
ti = ti + ji
else:
ti = ti - ji
time.sleep(ti / 1000)
def stop(self):
self._stop.set()
def close(self):
errc = Chan()
self.quit.put(errc)
result = errc.get()
self.thread.join(0.2)
assert not self.thread.is_alive()
return result
def __init__(self, fetcher):
self.fetcher = fetcher
self.updates_chan = Chan()
self.quit = Chan()
self.thread = threading.Thread(name='Subscription', target=self._run)
#self.thread.daemon = True
self.thread.start()
def test_buf_simple(self):
S = 5
c = Chan(S)
for i in range(S):
c.put(i)
c.close()
results = list(c)
self.assertEqual(results, list(range(S)))
def __init__(self, fetcher):
self.fetcher = fetcher
self.updates_chan = Chan()
self.quit = Chan()
self.thread = threading.Thread(
name='Subscription',
target=self._run)
#self.thread.daemon = True
self.thread.start()
def __init__(self, subscriptions):
self.subscriptions = subscriptions
self.updates_chan = Chan()
self.quit = Chan()
self.thread = threading.Thread(
name="Merged",
target=self._run)
self.thread.start()
def __init__(self, color, blink=False, period=0.5):
self.blink = blink
self.period = period
self.color = color.split('-')[0]
self.chan = Chan()
self._stop = threading.Event()
self._thread = threading.Thread(name='LightTimer', target=self.run)
self._thread.daemon = True
self._thread.start()
def __init__(self, period=DEFAULT_PERIOD, yellow=DEFAULT_YELLOW):
self.period = period
self.yellow = yellow
self.trip = False
self._stop = threading.Event()
self.chan = Chan()
self._thread = threading.Thread(name='LightTimer', target=self.run)
self._thread.daemon = True
self._thread.start()
def test_simple(self):
chan = Chan()
results = []
quickthread(accumulator, chan, results)
chan.put("Hello")
time.sleep(0.01) # Technically unsafe
self.assertEqual(len(results), 1)
self.assertEqual(results[0], "Hello")
def __init__(self, base=1000, jitter=500):
self.trip = False
self._stop = threading.Event()
self.chan = Chan()
self._thread = threading.Thread(name='LightTimer', target=self.run)
self._thread.daemon = True
self._thread.start()
self.base = base
self.jitter = jitter
random.seed()
def status(self):
st = {'server': 'ok'}
try:
resp_chan = Chan(1)
self.chan.put(('status', resp_chan), timeout=0.5)
resp = resp_chan.get(timeout=0.5)
st.update(resp)
except Timeout:
st['loop'] = 'Error: Did not hear from main thread in time'
return st
def status(self):
st = {'server': 'ok'}
try:
resp_chan = Chan(1)
self.chan.put(('status', resp_chan), timeout=0.5)
resp = resp_chan.get(timeout=0.5)
st.update(resp)
except Timeout:
st['loop'] = 'Error: Did not hear from main thread in time'
return st
def test_simple(self):
chan = Chan()
results = []
go(accumulator, chan, results)
chan.put("Hello")
time.sleep(0.01) # Technically unsafe
self.assertEqual(len(results), 1)
self.assertEqual(results[0], "Hello")
def test_select_timeout(self):
a = Chan()
b = Chan()
c = Chan()
self.assertRaises(Timeout, select, [a, b], [(c, 42)], timeout=0)
self.assertRaises(Timeout, select, [a, b], [(c, 42)], timeout=0.01)
# Verifies that select didn't leave any wishes lying around.
self.assertRaises(Timeout, a.put, 12, timeout=0)
self.assertRaises(Timeout, c.get, timeout=0)
class Subscription(object):
def __init__(self, fetcher):
self.fetcher = fetcher
self.updates_chan = Chan()
self.quit = Chan()
self.thread = threading.Thread(
name='Subscription',
target=self._run)
#self.thread.daemon = True
self.thread.start()
def _run(self):
next_time = time.time()
pending = [] # First is most recent. Should be a deque
err = None
while True:
start_fetch = timeout_after(max(0.0, next_time - time.time()))
# Does or doesn't wait on updates_chan depending on if we have
# items ready.
if pending:
outchans = [(self.updates_chan, pending[0])]
else:
outchans = []
ch, value = chanselect([self.quit, start_fetch], outchans)
if ch == self.quit:
errc = value
self.updates_chan.close()
errc.put(err)
return
elif ch == start_fetch:
try:
err = None
item_list, next_time = self.fetcher.fetch()
except Exception as ex:
err = ex
next_time = time.time() + 10.0
continue
pending.extend(item_list)
else: # self.updates_chan
pending.pop(0) # Pops the sent item
def updates(self):
return self.updates_chan
def close(self):
errc = Chan()
self.quit.put(errc)
result = errc.get()
self.thread.join(0.2)
assert not self.thread.is_alive()
return result
class Subscription(object):
def __init__(self, fetcher):
self.fetcher = fetcher
self.updates_chan = Chan()
self.quit = Chan()
self.thread = threading.Thread(name='Subscription', target=self._run)
#self.thread.daemon = True
self.thread.start()
def _run(self):
next_time = time.time()
pending = [] # First is most recent. Should be a deque
err = None
while True:
start_fetch = timeout_after(max(0.0, next_time - time.time()))
# Does or doesn't wait on updates_chan depending on if we have
# items ready.
if pending:
outchans = [(self.updates_chan, pending[0])]
else:
outchans = []
ch, value = chanselect([self.quit, start_fetch], outchans)
if ch == self.quit:
errc = value
self.updates_chan.close()
errc.put(err)
return
elif ch == start_fetch:
try:
err = None
item_list, next_time = self.fetcher.fetch()
except Exception as ex:
err = ex
next_time = time.time() + 10.0
continue
pending.extend(item_list)
else: # self.updates_chan
pending.pop(0) # Pops the sent item
def updates(self):
return self.updates_chan
def close(self):
errc = Chan()
self.quit.put(errc)
result = errc.get()
self.thread.join(0.2)
assert not self.thread.is_alive()
return result
def boring(msg):
c = Chan()
wait_for_it = Chan()
def sender():
i = 0
while True:
c.put(Message("%s: %d" % (msg, i), wait_for_it))
time.sleep(0.2 * random.random())
wait_for_it.get()
i += 1
go(sender)
return c
def test_buf_overfull(self):
c = Chan(5)
go(sayset, c, list(range(20)), delay=0)
time.sleep(0.1) # Fill up buffer
results = list(c)
self.assertEqual(results, list(range(20)))
def __init__(self, fn, In = None, Out = None):
if In is None: In = sys.stdin
if Out is None: Out = sys.stdout
self.fn = fn
self.chan = Chan()
self.chan.set_channels(In, Out)
self.filewatch = FileWatcher(fn)
self.code_locals = {}
try:
self.load_code(self.code_locals)
self.filewatch.note_new_files()
self.chan.send_cmd('ready')
except:
self.filewatch.note_new_files()
self.chan.send_cmd('exception', string = traceback.format_exc())
def test_putget_timeout(self):
c = Chan()
self.assertRaises(Timeout, c.put, 'x', timeout=0)
self.assertRaises(Timeout, c.put, 'x', timeout=0.01)
self.assertRaises(Timeout, c.get, timeout=0)
self.assertRaises(Timeout, c.get, timeout=0.01)
self.assertRaises(Timeout, c.put, 'x', timeout=0)
def __init__(self, args):
self.chan = Chan()
self._server = SimpleXMLRPCServer(('0.0.0.0', SERVER_PORT),
logRequests=False,
allow_none=True)
self.mode = args.mode
self._server.register_introspection_functions()
self._server.register_instance(self)
self._stop = threading.Event()
self.changed = 0
if args.controls:
self._thread = threading.Thread(name='Server', target=self.run)
else:
self._thread = threading.Thread(name='Server',
target=self._server.serve_forever)
self._thread.daemon = True
self._thread.start()
def timer(duration):
def timer_thread(chan, duration):
time.sleep(duration)
chan.put(time.time())
c = Chan()
go(timer_thread, c, duration)
return c
def __init__(self, period=DEFAULT_PERIOD, yellow=DEFAULT_YELLOW):
self.period = period
self.yellow = yellow
self.trip = False
self._stop = threading.Event()
self.chan = Chan()
self._thread = threading.Thread(name='LightTimer', target=self.run)
self._thread.daemon = True
self._thread.start()
def fan_in(*input_list):
def forward(input, output):
while True:
output.put(input.get())
c = Chan()
for input in input_list:
go(forward, input, c)
return c
def example_daisy():
def f(left, right):
left.put(1 + right.get())
N = 1000 # Python's threads aren't that lightweight
leftmost = Chan()
rightmost = leftmost
left = leftmost
for i in xrange(N):
right = Chan()
quickthread(f, left, right)
left = right
def putter():
right.put(1)
quickthread(putter)
print leftmost.get()
def fan_in(input1, input2):
def forwarder(input, output):
while True:
output.put(input.get())
c = Chan()
go(forwarder, input1, c)
go(forwarder, input2, c)
return c
def fan_in(input1, input2):
c = Chan()
def forward():
while True:
chan, value = chanselect([input1, input2], [])
c.put(value)
quickthread(forward)
return c
def example_daisy():
def f(left, right):
left.put(1 + right.get())
N = 1000 # Python's threads aren't that lightweight
leftmost = Chan()
rightmost = leftmost
left = leftmost
for i in xrange(N):
right = Chan()
go(f, left, right)
left = right
def putter():
right.put(1)
go(putter)
print leftmost.get()
def timeout_after(delay):
def thread(ch):
time.sleep(delay)
ch.put(None)
c = Chan()
t = threading.Thread(name='timeout', target=thread, args=(c, ))
t.daemon = True
t.start()
return c
def fan_in(input1, input2):
c = Chan()
def forward():
while True:
chan, value = select([input1, input2], [])
c.put(value)
go(forward)
return c
def __init__(self):
self.chan = Chan()
self._server = SimpleXMLRPCServer(
('0.0.0.0', SERVER_PORT),
logRequests=False,
allow_none=True)
self._server.register_introspection_functions()
self._server.register_instance(self)
self._thread = threading.Thread(name='Server', target=self._server.serve_forever)
self._thread.daemon = True
self._thread.start()
def boring(message):
def sender(message, c):
i = 0
while True:
c.put("%s: %d" % (message, i))
time.sleep(0.2 * random.random())
i += 1
c = Chan()
go(sender, message, c)
return c
def test_nothing_lost(self):
phrases = ['Hello_%03d' % x for x in range(1000)]
firstchan = Chan()
chan_layer1 = [Chan() for i in range(6)]
lastchan = Chan()
sayer = quickthread(sayset,
firstchan,
phrases,
delay=0.001,
__name='sayer')
# Distribute firstchan -> chan_layer1
for i in range(12):
outchans = [
chan_layer1[(i + j) % len(chan_layer1)] for j in range(3)
]
quickthread(distributer, [firstchan],
outchans,
delay_max=0.005,
__name='dist_layer1_%02d' % i)
# Distribute chan_layer1 -> lastchan
for i in range(12):
inchans = [
chan_layer1[(i + j) % len(chan_layer1)]
for j in range(0, 9, 3)
]
quickthread(distributer,
inchans, [lastchan],
delay_max=0.005,
__name='dist_layer2_%02d' % i)
results = []
quickthread(accumulator, lastchan, results, __name='accumulator')
sayer.join(10)
self.assertFalse(sayer.is_alive())
time.sleep(1) # Unsafe. Lets the data propagate to the accumulator
# Checks that none are missing, and there are no duplicates.
self.assertEqual(len(results), len(phrases))
self.assertEqual(set(results), set(phrases))
class Server(object):
def __init__(self, args):
self.chan = Chan()
self._server = SimpleXMLRPCServer(('0.0.0.0', SERVER_PORT),
logRequests=False,
allow_none=True)
self.mode = args.mode
self._server.register_introspection_functions()
self._server.register_instance(self)
self._stop = threading.Event()
self.changed = 0
if args.controls:
self._thread = threading.Thread(name='Server', target=self.run)
else:
self._thread = threading.Thread(name='Server',
target=self._server.serve_forever)
self._thread.daemon = True
self._thread.start()
def run(self):
while not self._stop.is_set():
mode = read_mode(self.mode)
now = int(round(time.time() * 1000))
if mode != self.mode and (now - self.changed > 250):
self.mode = mode
self.chan.put(('mode', self.mode), timeout=2.0)
self.changed = now
time.sleep(0.1)
def stop(self):
self._stop.set()
def status(self):
st = {'server': 'ok'}
try:
resp_chan = Chan(1)
self.chan.put(('status', resp_chan), timeout=0.5)
resp = resp_chan.get(timeout=0.5)
st.update(resp)
except Timeout:
st['loop'] = 'Error: Did not hear from main thread in time'
return st
def set_period(self, period):
self.chan.put(('set_period', period), timeout=2.0)
return True
def trip(self):
self.chan.put(('trip', ), timeout=2.0)
return True
def boring(msg):
c = Chan()
def sender():
i = 0
while True:
c.put("%s: %d" % (msg, i))
time.sleep(1.5 * random.random())
i += 1
go(sender)
return c
class Server(object):
def __init__(self):
self.chan = Chan()
self._server = SimpleXMLRPCServer(
('0.0.0.0', SERVER_PORT),
logRequests=False,
allow_none=True)
self._server.register_introspection_functions()
self._server.register_instance(self)
self._thread = threading.Thread(name='Server', target=self._server.serve_forever)
self._thread.daemon = True
self._thread.start()
def status(self):
st = {'server': 'ok'}
try:
resp_chan = Chan(1)
self.chan.put(('status', resp_chan), timeout=0.5)
resp = resp_chan.get(timeout=0.5)
st.update(resp)
except Timeout:
st['loop'] = 'Error: Did not hear from main thread in time'
return st
def set_period(self, period):
self.chan.put(('set_period', period), timeout=2.0)
return True
def trip(self):
self.chan.put(('trip',), timeout=2.0)
return True
def example_rcvquit():
def boring(msg, quit):
c = Chan()
def sender():
i = 0
while True:
time.sleep(1.0 * random.random())
chan, _ = chanselect([quit], [(c, "%s: %d" % (msg, i))])
if chan == quit:
quit.put("See you!")
i += 1
quickthread(sender)
return c
quit = Chan()
c = boring("Joe", quit)
for i in xrange(random.randint(0, 10), 0, -1):
print c.get()
quit.put("Bye!")
print "Joe says:", quit.get()
class LightTimer(object):
def __init__(self, period=DEFAULT_PERIOD, yellow=DEFAULT_YELLOW):
self.period = period
self.yellow = yellow
self.trip = False
self._stop = threading.Event()
self.chan = Chan()
self._thread = threading.Thread(name='LightTimer', target=self.run)
self._thread.daemon = True
self._thread.start()
def run(self):
STATES = [
[lights.RED1, lights.GRN2],
[lights.RED1, lights.YLW2],
[lights.GRN1, lights.RED2],
[lights.YLW1, lights.RED2],
]
state_i = len(STATES) - 1
t_last = 0
while not self._stop.is_set():
t_yellow = min(0.25 * self.period, DEFAULT_YELLOW)
t_green = self.period - t_yellow
dur = t_green if state_i % 2 == 0 else t_yellow
now = time.time()
if now > t_last + dur or self.trip:
t_last = now
state_i = (state_i + 1) % len(STATES)
self.chan.put(STATES[state_i])
self.trip = False
time.sleep(0.1)
self.chan.close()
def stop(self):
self._stop.set()
class Merged(object):
def __init__(self, subscriptions):
self.subscriptions = subscriptions
self.updates_chan = Chan()
self.quit = Chan()
self.thread = threading.Thread(
name="Merged",
target=self._run)
self.thread.start()
def _close_subs_collect_errs(self):
return [sub.close() for sub in self.subscriptions]
def _run(self):
subchans = [sub.updates() for sub in self.subscriptions]
while True:
c, value = chanselect(subchans + [self.quit], [])
if c == self.quit:
value.put(self._close_subs_collect_errs())
self.updates_chan.close()
return
else:
item = value
c, _ = chanselect([self.quit], [(self.updates_chan, item)])
if c == self.quit:
value.put(self._close_subs_collect_errs())
self.updates_chan.close()
return
else:
pass # Send successful
def updates(self):
return self.updates_chan
def close(self):
errc = Chan()
self.quit.put(errc)
result = errc.get()
self.thread.join(timeout=0.2)
assert not self.thread.is_alive()
return result
class Runner:
STOP = 0
CONTINUE = 1
def __init__(self, fn, In = None, Out = None):
if In is None: In = sys.stdin
if Out is None: Out = sys.stdout
self.fn = fn
self.chan = Chan()
self.chan.set_channels(In, Out)
self.filewatch = FileWatcher(fn)
self.code_locals = {}
try:
self.load_code(self.code_locals)
self.filewatch.note_new_files()
self.chan.send_cmd('ready')
except:
self.filewatch.note_new_files()
self.chan.send_cmd('exception', string = traceback.format_exc())
def run_cmd(self, cmd):
if cmd['action'] == 'shutdown':
return Runner.STOP
elif cmd['action'] == 'get_locals':
value = self.code_locals.get(cmd['var'], '')
self.chan.send_cmd('locals', val = value)
elif cmd['action'] == 'set_reload_interval':
self.filewatch.interval = cmd['value']
elif cmd['action'] == 'run_func':
self.run_func(cmd)
elif cmd['action'] == 'crash':
sys.exit()
elif cmd['action'] == 'abort':
# client disconnected, but script was finished by that time
pass
elif cmd['action'] == 'source_changed':
value = 1 if self.filewatch.has_changed() else 0
self.chan.send_cmd('', value = value)
else:
log('runner.py has idea was this incoming is:\n', repr(cmd))
raise NotImplementedError
return Runner.CONTINUE
def loop(self):
import time
while 1:
cmd = self.chan.recv_cmd()
#log("Test %d %s\n" % (time.time(), repr(cmd)))
if self.run_cmd(cmd) == Runner.STOP:
return
def load_code(self, my_locals):
with open(self.fn) as f:
code_src = f.read()
try:
self.code = compile(code_src, os.path.abspath(self.fn), 'exec')
except Exception, e:
hosts_line = re.findall('^(hosts\s*=[^\n]+)', code_src)
if hosts_line:
c = compile(hosts_line[0], '', 'exec')
exec c in my_locals
raise e
exec self.code in my_locals
def main(args):
"""
Backend for the Web interface to PMJQ.
Usage:
pmjq_sff [--exec=<exec.py>] --dl-folder=<folder>
[--root=<root>] --dl-url=<url> <eval>
Options:
--exec=<exec.py> Specify a Python file to be exec()d before <eval>
is eval()ed.
--root=<root> Working directory to evaluate other relative paths
from.
--dl-folder=<folder> Directory where the output files will be made
available to the users.
--dl-url=<url> Publicly accessible URL of <folder>
"""
root = os.path.normpath(abspath(args['--root'])) \
if args['--root'] is not None else None
if os.path.isabs(args['--dl-folder']):
dlfolder = args['--dl-folder']
elif root is not None:
dlfolder = pjoin(root, args['--dl-folder'])
else:
dlfolder = os.path.abspath(args('--dl-folder'))
dlurl = args['--dl-url']
nonce = str(random.randint(0, 10000)).zfill(4)
# The job id is the name of the file, the extension will come from the user
job_id = dt.now().isoformat() + "_" + nonce
inputs, outputs, logs = run_on_transitions_from_cli(
args, find_leaves, root=root)
for indir in inputs: # Write input files
assert ':' not in indir, \
"FIXME: (later) ':' in dir names not supported yet"
send("input:" + indir)
sys.stderr.write("Watching log files: ")
sys.stderr.write(str(logs)+"\n")
logch = Chan()
for logfile in logs:
go(tail_filter_and_process, fname=logfile,
filterfunc=get_default_filter(job_id),
handlerfunc=default_handler, chan=logch)
go(tail_filter_and_process, fname=logfile,
filterfunc=waiting_filter,
handlerfunc=waiting_handler,
root=root or "", chan=logch)
go(tail_filter_and_process, fname=logfile,
filterfunc=get_error_filter(job_id),
handlerfunc=error_handler,
root=root or "", chan=logch)
i = 0
ch = Chan()
sys.stderr.write("Watching output dirs: ")
sys.stderr.write(str(outputs)+"\n")
for outdir in outputs:
prefix = os.path.basename(os.path.normpath(outdir)) + str(i)
i += 1
go(move_and_print, outdir, job_id,
dlfolder, dlurl, ch, prefix)
go(handle_inputs, inputs, job_id)
# Wait for output to be processed
for _ in range(len(outputs)):
ch.get()
for _ in range(len(logs)*3):
logch.put(True)
for _ in range(len(logs)*3):
logch.get()
return
