class DiscoTimer(object): def __init__(self, base=1000, jitter=500): self.trip = False self._stop = threading.Event() self.chan = Chan() self._thread = threading.Thread(name='LightTimer', target=self.run) self._thread.daemon = True self._thread.start() self.base = base self.jitter = jitter random.seed() def run(self): while not self._stop.is_set(): random_lights = [] for l in lights.ALL: maybe = random.randint(0, 1) if maybe == 1: random_lights.append(l) self.chan.put(random_lights) d = random.randint(0, 1) ti = self.base ji = random.randint(0, self.jitter) if d == 0: ti = ti + ji else: ti = ti - ji time.sleep(ti / 1000) def stop(self): self._stop.set()
def test_buf_simple(self): S = 5 c = Chan(S) for i in range(S): c.put(i) c.close() results = list(c) self.assertEqual(results, list(range(S)))
def write_write_default(): """Two channels can be read and there's a default clause Expected output : ----------------- Writing (42|51) to ch(1|2) Writing (42|51) to ch(1|2) Received number (42|51) from ch(1|2) """ ch1A = Chan() ch1B = Chan() ch2A = Chan() ch2B = Chan() def read_channel1(): """Read the value in ch1A and write it in ch1B""" py_output.write('Reading from ch1\n'.encode()) x = ch1A.get() ch1B.put(x) def read_channel2(): """Read the value in ch2A and write it in ch2B""" py_output.write('Reading from ch2\n'.encode()) x = ch2A.get() ch2B.put(x) go(read_channel1) go(read_channel2) time.sleep(0.1) # Hopefully this is enough time to let # the other two goroutines block on their channel writes nb1 = 42 nb2 = 51 sent = 0 chan, nb = select(consumers=[], producers=[(ch1A, 42), (ch2A, 51)], default=True) if chan == ch1A: py_output.write('Sent number 42 to ch2\n'.encode()) sent = nb1 elif chan == ch2A: py_output.write('Sent number 51 to ch2\n'.encode()) sent = nb2 elif chan == 'default': raise Exception('Selected default behavior') if sent == 42: # ch1.put(42) happened ch2A.put(17) assert ch1B.get() == 42, "We should be able to read 42 from ch1B" elif sent == 51: # ch2.put(51) happened ch1A.put(17) assert ch2B.get() == 51, "We should be able to read 51 from ch2B" else: raise Exception('WTF ?')
def test_simple(self): chan = Chan() results = [] quickthread(accumulator, chan, results) chan.put("Hello") time.sleep(0.01) # Technically unsafe self.assertEqual(len(results), 1) self.assertEqual(results[0], "Hello")
def test_simple(self): chan = Chan() results = [] go(accumulator, chan, results) chan.put("Hello") time.sleep(0.01) # Technically unsafe self.assertEqual(len(results), 1) self.assertEqual(results[0], "Hello")
class Subscription(object): def __init__(self, fetcher): self.fetcher = fetcher self.updates_chan = Chan() self.quit = Chan() self.thread = threading.Thread( name='Subscription', target=self._run) #self.thread.daemon = True self.thread.start() def _run(self): next_time = time.time() pending = [] # First is most recent. Should be a deque err = None while True: start_fetch = timeout_after(max(0.0, next_time - time.time())) # Does or doesn't wait on updates_chan depending on if we have # items ready. if pending: outchans = [(self.updates_chan, pending[0])] else: outchans = [] ch, value = chanselect([self.quit, start_fetch], outchans) if ch == self.quit: errc = value self.updates_chan.close() errc.put(err) return elif ch == start_fetch: try: err = None item_list, next_time = self.fetcher.fetch() except Exception as ex: err = ex next_time = time.time() + 10.0 continue pending.extend(item_list) else: # self.updates_chan pending.pop(0) # Pops the sent item def updates(self): return self.updates_chan def close(self): errc = Chan() self.quit.put(errc) result = errc.get() self.thread.join(0.2) assert not self.thread.is_alive() return result
class Subscription(object): def __init__(self, fetcher): self.fetcher = fetcher self.updates_chan = Chan() self.quit = Chan() self.thread = threading.Thread(name='Subscription', target=self._run) #self.thread.daemon = True self.thread.start() def _run(self): next_time = time.time() pending = [] # First is most recent. Should be a deque err = None while True: start_fetch = timeout_after(max(0.0, next_time - time.time())) # Does or doesn't wait on updates_chan depending on if we have # items ready. if pending: outchans = [(self.updates_chan, pending[0])] else: outchans = [] ch, value = chanselect([self.quit, start_fetch], outchans) if ch == self.quit: errc = value self.updates_chan.close() errc.put(err) return elif ch == start_fetch: try: err = None item_list, next_time = self.fetcher.fetch() except Exception as ex: err = ex next_time = time.time() + 10.0 continue pending.extend(item_list) else: # self.updates_chan pending.pop(0) # Pops the sent item def updates(self): return self.updates_chan def close(self): errc = Chan() self.quit.put(errc) result = errc.get() self.thread.join(0.2) assert not self.thread.is_alive() return result
def read_write_default(): """One channel can be read, the written and there's a default clause Expected output : ----------------- (Writing 42 to ch1|Reading from ch2) (Writing 42 to ch1|Reading from ch2) (Received number 42 from ch1|Sent number 51 to ch2) """ ch1 = Chan() ch2 = Chan() def write_channel(): """Write 42 into ch1""" py_output.write('Writing 42 to ch1\n'.encode()) ch1.put(42) def read_channel(): """Read the value in ch2 and write it in ch1""" py_output.write('Reading from ch2\n'.encode()) x = ch2.get() ch1.put(x) go(write_channel) go(read_channel) time.sleep(0.1) # Hopefully this is enough time to let # the other two goroutines block on their channel writes nb = 51 chan, nb = select(consumers=[ch1], producers=[(ch2, 51)], default=True) if chan == ch1: py_output.write('Received number {} from ch1\n'.format(nb).encode()) elif chan == ch2: py_output.write('Sent number 51 to ch2\n'.encode()) elif chan == 'default': raise Exception('Selected default behavior') if nb == 42: # ch1.get() happened ch2.put(17) assert ch1.get() == 17, "We should be able to read 17 from ch1" elif nb is None: # ch2.put(51) happened assert ch1.get() == 42, "We should be able to read '42' from ch1" else: raise Exception('WTF ?')
class Merged(object): def __init__(self, subscriptions): self.subscriptions = subscriptions self.updates_chan = Chan() self.quit = Chan() self.thread = threading.Thread( name="Merged", target=self._run) self.thread.start() def _close_subs_collect_errs(self): return [sub.close() for sub in self.subscriptions] def _run(self): subchans = [sub.updates() for sub in self.subscriptions] while True: c, value = chanselect(subchans + [self.quit], []) if c == self.quit: value.put(self._close_subs_collect_errs()) self.updates_chan.close() return else: item = value c, _ = chanselect([self.quit], [(self.updates_chan, item)]) if c == self.quit: value.put(self._close_subs_collect_errs()) self.updates_chan.close() return else: pass # Send successful def updates(self): return self.updates_chan def close(self): errc = Chan() self.quit.put(errc) result = errc.get() self.thread.join(timeout=0.2) assert not self.thread.is_alive() return result
class Server(object): def __init__(self, args): self.chan = Chan() self._server = SimpleXMLRPCServer(('0.0.0.0', SERVER_PORT), logRequests=False, allow_none=True) self.mode = args.mode self._server.register_introspection_functions() self._server.register_instance(self) self._stop = threading.Event() self.changed = 0 if args.controls: self._thread = threading.Thread(name='Server', target=self.run) else: self._thread = threading.Thread(name='Server', target=self._server.serve_forever) self._thread.daemon = True self._thread.start() def run(self): while not self._stop.is_set(): mode = read_mode(self.mode) now = int(round(time.time() * 1000)) if mode != self.mode and (now - self.changed > 250): self.mode = mode self.chan.put(('mode', self.mode), timeout=2.0) self.changed = now time.sleep(0.1) def stop(self): self._stop.set() def status(self): st = {'server': 'ok'} try: resp_chan = Chan(1) self.chan.put(('status', resp_chan), timeout=0.5) resp = resp_chan.get(timeout=0.5) st.update(resp) except Timeout: st['loop'] = 'Error: Did not hear from main thread in time' return st def set_period(self, period): self.chan.put(('set_period', period), timeout=2.0) return True def trip(self): self.chan.put(('trip', ), timeout=2.0) return True
class Merged(object): def __init__(self, subscriptions): self.subscriptions = subscriptions self.updates_chan = Chan() self.quit = Chan() self.thread = threading.Thread(name="Merged", target=self._run) self.thread.start() def _close_subs_collect_errs(self): return [sub.close() for sub in self.subscriptions] def _run(self): subchans = [sub.updates() for sub in self.subscriptions] while True: c, value = chanselect(subchans + [self.quit], []) if c == self.quit: value.put(self._close_subs_collect_errs()) self.updates_chan.close() return else: item = value c, _ = chanselect([self.quit], [(self.updates_chan, item)]) if c == self.quit: value.put(self._close_subs_collect_errs()) self.updates_chan.close() return else: pass # Send successful def updates(self): return self.updates_chan def close(self): errc = Chan() self.quit.put(errc) result = errc.get() self.thread.join(timeout=0.2) assert not self.thread.is_alive() return result
class Server(object): def __init__(self): self.chan = Chan() self._server = SimpleXMLRPCServer( ('0.0.0.0', SERVER_PORT), logRequests=False, allow_none=True) self._server.register_introspection_functions() self._server.register_instance(self) self._thread = threading.Thread(name='Server', target=self._server.serve_forever) self._thread.daemon = True self._thread.start() def status(self): st = {'server': 'ok'} try: resp_chan = Chan(1) self.chan.put(('status', resp_chan), timeout=0.5) resp = resp_chan.get(timeout=0.5) st.update(resp) except Timeout: st['loop'] = 'Error: Did not hear from main thread in time' return st def set_period(self, period): self.chan.put(('set_period', period), timeout=2.0) return True def trip(self): self.chan.put(('trip',), timeout=2.0) return True
def read_default(): """The select should execute the default code as many times as needed (hopefully once) Then the <-ch1 case should happen Expected output : ----------------- Selected default behavior Writing 42 to ch1 Received number 42 from ch1 after 1[0-9].[0-9]*µs """ ch1 = Chan() ch2 = Chan() def write_channel(ch): """Wait for ch to be readable and Write an int nb into a channel ch""" ch.get() py_output.write('Writing 42 to ch1\n'.encode()) ch1.put(42) go(write_channel, ch2) from_ch1 = 0 wrote_to_ch2 = False start = time.time() while from_ch1 == 0: chan, nb = select(consumers=[ch1], producers=[], default=True) if chan == ch1: elapsed = time.time() - start from_ch1 = nb py_output.write('Received number ' '{} from ch1 after ' '{}\n'.format(from_ch1, elapsed).encode()) if chan is 'default': py_output.write('Selected default behavior\n'.encode()) if not wrote_to_ch2: ch2.put(1) wrote_to_ch2 = True
def example_rcvquit(): def boring(msg, quit): c = Chan() def sender(): i = 0 while True: time.sleep(1.0 * random.random()) chan, _ = chanselect([quit], [(c, "%s: %d" % (msg, i))]) if chan == quit: quit.put("See you!") i += 1 quickthread(sender) return c quit = Chan() c = boring("Joe", quit) for i in xrange(random.randint(0, 10), 0, -1): print c.get() quit.put("Bye!") print "Joe says:", quit.get()
def example_rcvquit(): def boring(msg, quit): c = Chan() def sender(): i = 0 while True: time.sleep(1.0 * random.random()) chan, _ = select([quit], [(c, "%s: %d" % (msg, i))]) if chan == quit: quit.put("See you!") i += 1 go(sender) return c quit = Chan() c = boring("Joe", quit) for i in xrange(random.randint(0, 10), 0, -1): print c.get() quit.put("Bye!") print "Joe says:", quit.get()
class LightTimer(object): def __init__(self, period=DEFAULT_PERIOD, yellow=DEFAULT_YELLOW): self.period = period self.yellow = yellow self.trip = False self._stop = threading.Event() self.chan = Chan() self._thread = threading.Thread(name='LightTimer', target=self.run) self._thread.daemon = True self._thread.start() def run(self): STATES = [ [lights.RED1, lights.GRN2], [lights.RED1, lights.YLW2], [lights.GRN1, lights.RED2], [lights.YLW1, lights.RED2], ] state_i = len(STATES) - 1 t_last = 0 while not self._stop.is_set(): t_yellow = min(0.25 * self.period, DEFAULT_YELLOW) t_green = self.period - t_yellow dur = t_green if state_i % 2 == 0 else t_yellow now = time.time() if now > t_last + dur or self.trip: t_last = now state_i = (state_i + 1) % len(STATES) self.chan.put(STATES[state_i]) self.trip = False time.sleep(0.1) self.chan.close() def stop(self): self._stop.set()
class SolidTimer(object): def __init__(self, color, blink=False, period=0.5): self.blink = blink self.period = period self.color = color.split('-')[0] self.chan = Chan() self._stop = threading.Event() self._thread = threading.Thread(name='LightTimer', target=self.run) self._thread.daemon = True self._thread.start() def run(self): STATE = [] if self.color == 'red': STATE = [lights.RED1, lights.RED2] elif self.color == 'yellow': STATE = [lights.YLW1, lights.YLW2] elif self.color == 'green': STATE = [lights.GRN1, lights.GRN2] on = True while not self._stop.is_set(): if on: if self.blink: self.chan.put([STATE[0]]) on = False time.sleep(self.period) else: self.chan.put(STATE) time.sleep(5) else: self.chan.put([STATE[1]]) on = True time.sleep(self.period) self.chan.close() def stop(self): self._stop.set()
def main(args): """ Backend for the Web interface to PMJQ. Usage: pmjq_sff [--exec=<exec.py>] --dl-folder=<folder> [--root=<root>] --dl-url=<url> <eval> Options: --exec=<exec.py> Specify a Python file to be exec()d before <eval> is eval()ed. --root=<root> Working directory to evaluate other relative paths from. --dl-folder=<folder> Directory where the output files will be made available to the users. --dl-url=<url> Publicly accessible URL of <folder> """ root = os.path.normpath(abspath(args['--root'])) \ if args['--root'] is not None else None if os.path.isabs(args['--dl-folder']): dlfolder = args['--dl-folder'] elif root is not None: dlfolder = pjoin(root, args['--dl-folder']) else: dlfolder = os.path.abspath(args('--dl-folder')) dlurl = args['--dl-url'] nonce = str(random.randint(0, 10000)).zfill(4) # The job id is the name of the file, the extension will come from the user job_id = dt.now().isoformat() + "_" + nonce inputs, outputs, logs = run_on_transitions_from_cli( args, find_leaves, root=root) for indir in inputs: # Write input files assert ':' not in indir, \ "FIXME: (later) ':' in dir names not supported yet" send("input:" + indir) sys.stderr.write("Watching log files: ") sys.stderr.write(str(logs)+"\n") logch = Chan() for logfile in logs: go(tail_filter_and_process, fname=logfile, filterfunc=get_default_filter(job_id), handlerfunc=default_handler, chan=logch) go(tail_filter_and_process, fname=logfile, filterfunc=waiting_filter, handlerfunc=waiting_handler, root=root or "", chan=logch) go(tail_filter_and_process, fname=logfile, filterfunc=get_error_filter(job_id), handlerfunc=error_handler, root=root or "", chan=logch) i = 0 ch = Chan() sys.stderr.write("Watching output dirs: ") sys.stderr.write(str(outputs)+"\n") for outdir in outputs: prefix = os.path.basename(os.path.normpath(outdir)) + str(i) i += 1 go(move_and_print, outdir, job_id, dlfolder, dlurl, ch, prefix) go(handle_inputs, inputs, job_id) # Wait for output to be processed for _ in range(len(outputs)): ch.get() for _ in range(len(logs)*3): logch.put(True) for _ in range(len(logs)*3): logch.get() return