def write_write_default(): """Two channels can be read and there's a default clause Expected output : ----------------- Writing (42|51) to ch(1|2) Writing (42|51) to ch(1|2) Received number (42|51) from ch(1|2) """ ch1A = Chan() ch1B = Chan() ch2A = Chan() ch2B = Chan() def read_channel1(): """Read the value in ch1A and write it in ch1B""" py_output.write('Reading from ch1\n'.encode()) x = ch1A.get() ch1B.put(x) def read_channel2(): """Read the value in ch2A and write it in ch2B""" py_output.write('Reading from ch2\n'.encode()) x = ch2A.get() ch2B.put(x) go(read_channel1) go(read_channel2) time.sleep(0.1) # Hopefully this is enough time to let # the other two goroutines block on their channel writes nb1 = 42 nb2 = 51 sent = 0 chan, nb = select(consumers=[], producers=[(ch1A, 42), (ch2A, 51)], default=True) if chan == ch1A: py_output.write('Sent number 42 to ch2\n'.encode()) sent = nb1 elif chan == ch2A: py_output.write('Sent number 51 to ch2\n'.encode()) sent = nb2 elif chan == 'default': raise Exception('Selected default behavior') if sent == 42: # ch1.put(42) happened ch2A.put(17) assert ch1B.get() == 42, "We should be able to read 42 from ch1B" elif sent == 51: # ch2.put(51) happened ch1A.put(17) assert ch2B.get() == 51, "We should be able to read 51 from ch2B" else: raise Exception('WTF ?')
def play(self): '''round-robin''' self.open_start_urls() loops_without_progress = 0 while True: import time start = time.time() if len(self.bots) == 0: return # bots got stuck if there's 2 wait pages in a row if loops_without_progress > 10: raise AssertionError('Bots got stuck') results = Chan(buflen=len(self.bots)) threads = [] for bot in self.bots.values(): if bot.on_wait_page(): pass else: thread = SubmitThread(bot, results) threads.append(thread) thread.start() for thread in threads: bot, status = results.get() if isinstance(status, Exception): raise status elif status == 'finished': del self.bots[bot.participant.id] else: bot.submit(status) results.close() if not threads: loops_without_progress += 1
def close(self): errc = Chan() self.quit.put(errc) result = errc.get() self.thread.join(0.2) assert not self.thread.is_alive() return result
def read_write_default(): """One channel can be read, the written and there's a default clause Expected output : ----------------- (Writing 42 to ch1|Reading from ch2) (Writing 42 to ch1|Reading from ch2) (Received number 42 from ch1|Sent number 51 to ch2) """ ch1 = Chan() ch2 = Chan() def write_channel(): """Write 42 into ch1""" py_output.write('Writing 42 to ch1\n'.encode()) ch1.put(42) def read_channel(): """Read the value in ch2 and write it in ch1""" py_output.write('Reading from ch2\n'.encode()) x = ch2.get() ch1.put(x) go(write_channel) go(read_channel) time.sleep(0.1) # Hopefully this is enough time to let # the other two goroutines block on their channel writes nb = 51 chan, nb = select(consumers=[ch1], producers=[(ch2, 51)], default=True) if chan == ch1: py_output.write('Received number {} from ch1\n'.format(nb).encode()) elif chan == ch2: py_output.write('Sent number 51 to ch2\n'.encode()) elif chan == 'default': raise Exception('Selected default behavior') if nb == 42: # ch1.get() happened ch2.put(17) assert ch1.get() == 17, "We should be able to read 17 from ch1" elif nb is None: # ch2.put(51) happened assert ch1.get() == 42, "We should be able to read '42' from ch1" else: raise Exception('WTF ?')
def example_daisy(): def f(left, right): left.put(1 + right.get()) N = 1000 # Python's threads aren't that lightweight leftmost = Chan() rightmost = leftmost left = leftmost for i in xrange(N): right = Chan() quickthread(f, left, right) left = right def putter(): right.put(1) quickthread(putter) print leftmost.get()
def status(self): st = {'server': 'ok'} try: resp_chan = Chan(1) self.chan.put(('status', resp_chan), timeout=0.5) resp = resp_chan.get(timeout=0.5) st.update(resp) except Timeout: st['loop'] = 'Error: Did not hear from main thread in time' return st
def example_daisy(): def f(left, right): left.put(1 + right.get()) N = 1000 # Python's threads aren't that lightweight leftmost = Chan() rightmost = leftmost left = leftmost for i in xrange(N): right = Chan() go(f, left, right) left = right def putter(): right.put(1) go(putter) print leftmost.get()
def read_read_default(): """Two channels can be read and there's a default clause Expected output : ----------------- Writing (42|51) to ch(1|2) Writing (42|51) to ch(1|2) Received number (42|51) from ch(1|2) """ ch1 = Chan() ch2 = Chan() def write_channel1(): """Write 42 into ch1""" py_output.write('Writing 42 to ch1\n'.encode()) ch1.put(42) def write_channel2(): """Write 51 into ch2""" py_output.write('Writing 51 to ch2\n'.encode()) ch2.put(51) go(write_channel1) go(write_channel2) time.sleep(0.1) # Hopefully this is enough time to let # the other two goroutines block on their channel writes chan, nb = select(consumers=[ch1, ch2], producers=[], default=True) if chan == ch1: py_output.write('Received number {} from ch1\n'.format(nb).encode()) elif chan == ch2: py_output.write('Received number {} from ch2\n'.format(nb).encode()) elif chan == 'default': raise Exception('Selected default behavior') if nb == 42: # ch1.get() happened assert ch2.get() == 51, "We should be able to read '51' from ch2" elif nb == 51: # ch2.get() happened assert ch1.get() == 42, "We should be able to read '42' from ch1" else: raise Exception('WTF ?')
def example_rcvquit(): def boring(msg, quit): c = Chan() def sender(): i = 0 while True: time.sleep(1.0 * random.random()) chan, _ = chanselect([quit], [(c, "%s: %d" % (msg, i))]) if chan == quit: quit.put("See you!") i += 1 quickthread(sender) return c quit = Chan() c = boring("Joe", quit) for i in xrange(random.randint(0, 10), 0, -1): print c.get() quit.put("Bye!") print "Joe says:", quit.get()
def example_rcvquit(): def boring(msg, quit): c = Chan() def sender(): i = 0 while True: time.sleep(1.0 * random.random()) chan, _ = select([quit], [(c, "%s: %d" % (msg, i))]) if chan == quit: quit.put("See you!") i += 1 go(sender) return c quit = Chan() c = boring("Joe", quit) for i in xrange(random.randint(0, 10), 0, -1): print c.get() quit.put("Bye!") print "Joe says:", quit.get()
def main(args): """ Backend for the Web interface to PMJQ. Usage: pmjq_sff [--exec=<exec.py>] --dl-folder=<folder> [--root=<root>] --dl-url=<url> <eval> Options: --exec=<exec.py> Specify a Python file to be exec()d before <eval> is eval()ed. --root=<root> Working directory to evaluate other relative paths from. --dl-folder=<folder> Directory where the output files will be made available to the users. --dl-url=<url> Publicly accessible URL of <folder> """ root = os.path.normpath(abspath(args['--root'])) \ if args['--root'] is not None else None if os.path.isabs(args['--dl-folder']): dlfolder = args['--dl-folder'] elif root is not None: dlfolder = pjoin(root, args['--dl-folder']) else: dlfolder = os.path.abspath(args('--dl-folder')) dlurl = args['--dl-url'] nonce = str(random.randint(0, 10000)).zfill(4) # The job id is the name of the file, the extension will come from the user job_id = dt.now().isoformat() + "_" + nonce inputs, outputs, logs = run_on_transitions_from_cli( args, find_leaves, root=root) for indir in inputs: # Write input files assert ':' not in indir, \ "FIXME: (later) ':' in dir names not supported yet" send("input:" + indir) sys.stderr.write("Watching log files: ") sys.stderr.write(str(logs)+"\n") logch = Chan() for logfile in logs: go(tail_filter_and_process, fname=logfile, filterfunc=get_default_filter(job_id), handlerfunc=default_handler, chan=logch) go(tail_filter_and_process, fname=logfile, filterfunc=waiting_filter, handlerfunc=waiting_handler, root=root or "", chan=logch) go(tail_filter_and_process, fname=logfile, filterfunc=get_error_filter(job_id), handlerfunc=error_handler, root=root or "", chan=logch) i = 0 ch = Chan() sys.stderr.write("Watching output dirs: ") sys.stderr.write(str(outputs)+"\n") for outdir in outputs: prefix = os.path.basename(os.path.normpath(outdir)) + str(i) i += 1 go(move_and_print, outdir, job_id, dlfolder, dlurl, ch, prefix) go(handle_inputs, inputs, job_id) # Wait for output to be processed for _ in range(len(outputs)): ch.get() for _ in range(len(logs)*3): logch.put(True) for _ in range(len(logs)*3): logch.get() return