def test_buf_overfull(self): c = Chan(5) go(sayset, c, list(range(20)), delay=0) time.sleep(0.1) # Fill up buffer results = list(c) self.assertEqual(results, list(range(20)))
def timer(duration): def timer_thread(chan, duration): time.sleep(duration) chan.put(time.time()) c = Chan() go(timer_thread, c, duration) return c
def write_write_default(): """Two channels can be read and there's a default clause Expected output : ----------------- Writing (42|51) to ch(1|2) Writing (42|51) to ch(1|2) Received number (42|51) from ch(1|2) """ ch1A = Chan() ch1B = Chan() ch2A = Chan() ch2B = Chan() def read_channel1(): """Read the value in ch1A and write it in ch1B""" py_output.write('Reading from ch1\n'.encode()) x = ch1A.get() ch1B.put(x) def read_channel2(): """Read the value in ch2A and write it in ch2B""" py_output.write('Reading from ch2\n'.encode()) x = ch2A.get() ch2B.put(x) go(read_channel1) go(read_channel2) time.sleep(0.1) # Hopefully this is enough time to let # the other two goroutines block on their channel writes nb1 = 42 nb2 = 51 sent = 0 chan, nb = select(consumers=[], producers=[(ch1A, 42), (ch2A, 51)], default=True) if chan == ch1A: py_output.write('Sent number 42 to ch2\n'.encode()) sent = nb1 elif chan == ch2A: py_output.write('Sent number 51 to ch2\n'.encode()) sent = nb2 elif chan == 'default': raise Exception('Selected default behavior') if sent == 42: # ch1.put(42) happened ch2A.put(17) assert ch1B.get() == 42, "We should be able to read 42 from ch1B" elif sent == 51: # ch2.put(51) happened ch1A.put(17) assert ch2B.get() == 51, "We should be able to read 51 from ch2B" else: raise Exception('WTF ?')
def fan_in(input1, input2): def forwarder(input, output): while True: output.put(input.get()) c = Chan() go(forwarder, input1, c) go(forwarder, input2, c) return c
def fan_in(*input_list): def forward(input, output): while True: output.put(input.get()) c = Chan() for input in input_list: go(forward, input, c) return c
def test_simple(self): chan = Chan() results = [] go(accumulator, chan, results) chan.put("Hello") time.sleep(0.01) # Technically unsafe self.assertEqual(len(results), 1) self.assertEqual(results[0], "Hello")
def fan_in(input1, input2): c = Chan() def forward(): while True: chan, value = select([input1, input2], []) c.put(value) go(forward) return c
def test_select_and_closed(self): a, b, c = [Chan() for _ in range(3)] out = Chan() go(sayset, a, [0, 1, 2], delay=0.01, __name='sayset1') go(sayset, b, [3, 4, 5], delay=0.01, __name='sayset2') go(sayset, c, [6, 7, 8], delay=0.01, __name='sayset2') def fanin_until_closed(inchans, outchan): inchans = inchans[:] while inchans: try: _, val = select(inchans, []) out.put(val) except ChanClosed as ex: inchans.remove(ex.which) out.close() go(fanin_until_closed, [a, b, c], out, __name='fanin') into = [] acc = go(accumulator, out, into) acc.join(10) self.assertFalse(acc.is_alive()) results = set(into) self.assertEqual(len(results), 9) self.assertEqual(results, set(range(9)))
def boring(message): def sender(message, c): i = 0 while True: c.put("%s: %d" % (message, i)) time.sleep(0.2 * random.random()) i += 1 c = Chan() go(sender, message, c) return c
def boring(msg): c = Chan() def sender(): i = 0 while True: c.put("%s: %d" % (msg, i)) time.sleep(1.5 * random.random()) i += 1 go(sender) return c
def boring(msg): c = Chan() wait_for_it = Chan() def sender(): i = 0 while True: c.put(Message("%s: %d" % (msg, i), wait_for_it)) time.sleep(0.2 * random.random()) wait_for_it.get() i += 1 go(sender) return c
def boring(msg, quit): c = Chan() def sender(): i = 0 while True: time.sleep(1.0 * random.random()) chan, _ = select([quit], [(c, "%s: %d" % (msg, i))]) if chan == quit: quit.put("See you!") i += 1 go(sender) return c
def test_iter_and_closed(self): c = Chan() go(sayset, c, [1, 2, 3], delay=0) def listener(): it = iter(c) self.assertEqual(next(it), 1) self.assertEqual(next(it), 2) self.assertEqual(next(it), 3) self.assertRaises(StopIteration, it.__next__) t = go(listener) time.sleep(0.1) self.assertFalse(t.is_alive())
def read_write_default(): """One channel can be read, the written and there's a default clause Expected output : ----------------- (Writing 42 to ch1|Reading from ch2) (Writing 42 to ch1|Reading from ch2) (Received number 42 from ch1|Sent number 51 to ch2) """ ch1 = Chan() ch2 = Chan() def write_channel(): """Write 42 into ch1""" py_output.write('Writing 42 to ch1\n'.encode()) ch1.put(42) def read_channel(): """Read the value in ch2 and write it in ch1""" py_output.write('Reading from ch2\n'.encode()) x = ch2.get() ch1.put(x) go(write_channel) go(read_channel) time.sleep(0.1) # Hopefully this is enough time to let # the other two goroutines block on their channel writes nb = 51 chan, nb = select(consumers=[ch1], producers=[(ch2, 51)], default=True) if chan == ch1: py_output.write('Received number {} from ch1\n'.format(nb).encode()) elif chan == ch2: py_output.write('Sent number 51 to ch2\n'.encode()) elif chan == 'default': raise Exception('Selected default behavior') if nb == 42: # ch1.get() happened ch2.put(17) assert ch1.get() == 17, "We should be able to read 17 from ch1" elif nb is None: # ch2.put(51) happened assert ch1.get() == 42, "We should be able to read '42' from ch1" else: raise Exception('WTF ?')
def example_daisy(): def f(left, right): left.put(1 + right.get()) N = 1000 # Python's threads aren't that lightweight leftmost = Chan() rightmost = leftmost left = leftmost for i in xrange(N): right = Chan() go(f, left, right) left = right def putter(): right.put(1) go(putter) print leftmost.get()
def read_read_default(): """Two channels can be read and there's a default clause Expected output : ----------------- Writing (42|51) to ch(1|2) Writing (42|51) to ch(1|2) Received number (42|51) from ch(1|2) """ ch1 = Chan() ch2 = Chan() def write_channel1(): """Write 42 into ch1""" py_output.write('Writing 42 to ch1\n'.encode()) ch1.put(42) def write_channel2(): """Write 51 into ch2""" py_output.write('Writing 51 to ch2\n'.encode()) ch2.put(51) go(write_channel1) go(write_channel2) time.sleep(0.1) # Hopefully this is enough time to let # the other two goroutines block on their channel writes chan, nb = select(consumers=[ch1, ch2], producers=[], default=True) if chan == ch1: py_output.write('Received number {} from ch1\n'.format(nb).encode()) elif chan == ch2: py_output.write('Received number {} from ch2\n'.format(nb).encode()) elif chan == 'default': raise Exception('Selected default behavior') if nb == 42: # ch1.get() happened assert ch2.get() == 51, "We should be able to read '51' from ch2" elif nb == 51: # ch2.get() happened assert ch1.get() == 42, "We should be able to read '42' from ch1" else: raise Exception('WTF ?')
def read_default(): """The select should execute the default code as many times as needed (hopefully once) Then the <-ch1 case should happen Expected output : ----------------- Selected default behavior Writing 42 to ch1 Received number 42 from ch1 after 1[0-9].[0-9]*µs """ ch1 = Chan() ch2 = Chan() def write_channel(ch): """Wait for ch to be readable and Write an int nb into a channel ch""" ch.get() py_output.write('Writing 42 to ch1\n'.encode()) ch1.put(42) go(write_channel, ch2) from_ch1 = 0 wrote_to_ch2 = False start = time.time() while from_ch1 == 0: chan, nb = select(consumers=[ch1], producers=[], default=True) if chan == ch1: elapsed = time.time() - start from_ch1 = nb py_output.write('Received number ' '{} from ch1 after ' '{}\n'.format(from_ch1, elapsed).encode()) if chan is 'default': py_output.write('Selected default behavior\n'.encode()) if not wrote_to_ch2: ch2.put(1) wrote_to_ch2 = True
def main(): FetcherCls = MockFetcher merged = Merged([ Subscription(FetcherCls('blog.golang.org')), Subscription(FetcherCls('googleblog.blogspot.com')), Subscription(FetcherCls('googledevelopers.blogspot.com')) ]) # Close after a while def close_later(): time.sleep(3) print("Closed: {}".format(merged.close())) go(close_later) for it in merged.updates(): print("{} -- {}".format(it.channel, it.title)) time.sleep(0.1) print("Still active: (should only be _MainThread and timeouts)") for active in threading._active.itervalues(): print(" {}".format(active))
def read(): """The select should block, Then, the <- ch1 case should be executed very soon after the other goroutine times out. Expected output : ----------------- Writing 42 to ch1 Waited 1.00[0-9]*s """ ch1 = Chan() def write_channel_delayed(): """Write an int nb into a channel ch with a delay of 1 second""" time.sleep(1) py_output.write('Writing 42 to ch1\n'.encode()) ch1.put(42) go(write_channel_delayed) start = time.time() chan, nb = select(consumers=[ch1], producers=[]) if chan == ch1: py_output.write('Waited {}\n'.format(time.time() - start).encode())
def test_nothing_lost(self): phrases = ['Hello_%03d' % x for x in range(1000)] firstchan = Chan() chan_layer1 = [Chan() for i in range(6)] lastchan = Chan() sayer = go(sayset, firstchan, phrases, delay=0.001, __name='sayer') # Distribute firstchan -> chan_layer1 for i in range(12): outchans = [ chan_layer1[(i + j) % len(chan_layer1)] for j in range(3) ] go(distributer, [firstchan], outchans, delay_max=0.005, __name='dist_layer1_%02d' % i) # Distribute chan_layer1 -> lastchan for i in range(12): inchans = [ chan_layer1[(i + j) % len(chan_layer1)] for j in range(0, 9, 3) ] go(distributer, inchans, [lastchan], delay_max=0.005, __name='dist_layer2_%02d' % i) results = [] go(accumulator, lastchan, results, __name='accumulator') sayer.join(10) self.assertFalse(sayer.is_alive()) time.sleep(1) # Unsafe. Lets the data propagate to the accumulator # Checks that none are missing, and there are no duplicates. self.assertEqual(len(results), len(phrases)) self.assertEqual(set(results), set(phrases))
def main(args): """ Backend for the Web interface to PMJQ. Usage: pmjq_sff [--exec=<exec.py>] --dl-folder=<folder> [--root=<root>] --dl-url=<url> <eval> Options: --exec=<exec.py> Specify a Python file to be exec()d before <eval> is eval()ed. --root=<root> Working directory to evaluate other relative paths from. --dl-folder=<folder> Directory where the output files will be made available to the users. --dl-url=<url> Publicly accessible URL of <folder> """ root = os.path.normpath(abspath(args['--root'])) \ if args['--root'] is not None else None if os.path.isabs(args['--dl-folder']): dlfolder = args['--dl-folder'] elif root is not None: dlfolder = pjoin(root, args['--dl-folder']) else: dlfolder = os.path.abspath(args('--dl-folder')) dlurl = args['--dl-url'] nonce = str(random.randint(0, 10000)).zfill(4) # The job id is the name of the file, the extension will come from the user job_id = dt.now().isoformat() + "_" + nonce inputs, outputs, logs = run_on_transitions_from_cli( args, find_leaves, root=root) for indir in inputs: # Write input files assert ':' not in indir, \ "FIXME: (later) ':' in dir names not supported yet" send("input:" + indir) sys.stderr.write("Watching log files: ") sys.stderr.write(str(logs)+"\n") logch = Chan() for logfile in logs: go(tail_filter_and_process, fname=logfile, filterfunc=get_default_filter(job_id), handlerfunc=default_handler, chan=logch) go(tail_filter_and_process, fname=logfile, filterfunc=waiting_filter, handlerfunc=waiting_handler, root=root or "", chan=logch) go(tail_filter_and_process, fname=logfile, filterfunc=get_error_filter(job_id), handlerfunc=error_handler, root=root or "", chan=logch) i = 0 ch = Chan() sys.stderr.write("Watching output dirs: ") sys.stderr.write(str(outputs)+"\n") for outdir in outputs: prefix = os.path.basename(os.path.normpath(outdir)) + str(i) i += 1 go(move_and_print, outdir, job_id, dlfolder, dlurl, ch, prefix) go(handle_inputs, inputs, job_id) # Wait for output to be processed for _ in range(len(outputs)): ch.get() for _ in range(len(logs)*3): logch.put(True) for _ in range(len(logs)*3): logch.get() return
def test_buf_kept_empty(self): c = Chan(5) go(sayset, c, list(range(20)), delay=0.02) results = list(c) self.assertEqual(results, list(range(20)))