def do_test(test_func): global q, finished q = circular_fifo() finished = 0 coro.spawn(test_func) while not finished: coro.sleep_relative(1)
def sleep(x): """Sleep that works in both coro and non-coro environments.""" if GOT_CORO and coro.coro_is_running(): coro.sleep_relative(x) else: time.sleep(x)
def main(): pid, fi, fo, fe = process.spawn_job_bg('/bin/sleep 3') coro.wait_for_kevent (pid, kqueue_events.EV_ADD | kqueue_events.EV_ONESHOT, kqueue_events.EVFILT_PROC, kqueue_events.NOTE_EXIT) print 'wait done' os.waitpid(pid, 0) coro.sleep_relative(5) coro._exit = 1
def go (G): global in_flight addr0 = ('127.0.0.1', 0) addr1 = parse_addr_arg (G.args.connect) cp = ChainPuller (G, addr0, addr1) cp.wait_for ('verack') if G.args.getblocks: W ('downloading block names...\n') names = cp.getblocks() else: W ('downloading headers...\n') names = cp.getheaders() start = time.time() W ('got %d names...\n' % (len(names),)) WB ('[BLOCKNUM] ') WR ('(MB/s)\n') for i in range (0, len (names), 50): chunk = names[i:i+50] cp.getdata ([(OBJ_BLOCK, name) for name in chunk]) in_flight += len(chunk) in_flight_cv.wait() # let it finish while 1: coro.sleep_relative (1) if in_flight == 0: break stop = time.time() WB ( '\ntransferred %.2f GB in %d secs (%.2f MB/s)\n' % ( float(total_bytes) / GB, stop - start, (float(total_bytes) / MB) / (stop - start) ) ) coro.set_exit()
def watcher(thread_ids): while True: if len(thread_ids) == 0: coro.set_exit() break thread_ids = [x for x in thread_ids if x in coro.all_threads] coro.sleep_relative(0.1)
def pusher(): i = 0 while i < 100: the_fifo.push (i) coro.sleep_relative (.005) i += 1 coro._exit = 1
def start_aio(self): self._start_time = coro.get_now() self.main_thread_state = 'Starting writers.' self.log(1, 'Starting %i writers.', self.options.num_writers) for x in xrange(self.options.num_writers): self._worker_semaphore.acquire() coro.spawn(self._writer, x) while 1: # Spin lock. status = 'Waiting for writers to ramp up %i/%i.' % (self._writes_finished, self.options.reader_delay) self.main_thread_state = status self.log(2, status) if int(self._worker_semaphore) == 0: self.log(0, 'EEP! All writers exited too fast.') self.main_thread_state = 'Aborted.' return if self._writes_finished < self.options.reader_delay: coro.sleep_relative(0.5) else: break self.main_thread_state = 'Starting readers.' self.log(1, 'Starting %i readers.', self.options.num_readers) for x in xrange(self.options.num_readers): self._worker_semaphore.acquire() coro.spawn(self._reader, x) self.main_thread_state = 'Waiting for all threads to finish.' self.log(1, 'Waiting till all threads finished.') self._worker_semaphore.block_till_zero() self.main_thread_state = 'Done.'
def start (self, addr, retries=5): """Start the web server listening on addr in a new coroutine. Try up to retries time to bind to that address. Raises an exception if the bind fails.""" server_s = self._make_socket() done = 0 save_errno = 0 self.addr = addr while not done: for x in xrange (retries): try: was_eaddrinuse = 0 server_s.bind (addr) except OSError, why: if why.errno not in (errno.EADDRNOTAVAIL, errno.EADDRINUSE): raise else: save_errno = 0 if why.errno == errno.EADDRINUSE: was_eaddrinuse = 1 else: done = 1 break coro.sleep_relative(1) # ... and retry else: coro.print_stderr ("cannot bind to %s:%d after 5 attempts, errno = %d\n" % (addr[0], addr[1], save_errno)) if was_eaddrinuse: qlog.write('WEBUI.PORT_IN_USE', addr[0], str(addr[1])) coro.sleep_relative(15)
def go (self): # main hoovering thread. # first, get a list of blocks we need to fetch via getheaders. db = G.block_db if not db.num_block: self.queue.push (block_db.genesis_block_hash) self.qset.add (block_db.genesis_block_hash) try: self.running = True # get the list of all blocks we need to fetch... while 1: c = self.get_live_connection() G.log ('getheaders', 'start') t0 = block_db.timer() names = c.getheaders() G.log ('getheaders', 'stop', '%.02f' % t0.end()) if names: break else: coro.sleep_relative (5) # start fetching them... coro.spawn (self.drain_queue_thread) for name in names: self.in_flight_sem.acquire(1) self.queue.push (name) self.qset.add (name) self.queue.push (None) finally: self.running = False
def _get_to_write(self): attempts = 0 while 1: # Pick a random location to read from. attempts += 1 if not attempts % 3: self.log(2, 'Failed to pick write location 3 times.') coro.sleep_relative(0) block_pos = random.randint(1, self.options.blocks)-1 pos = block_pos * self.options.block_size if self.options.lio: max_lio_blocks = self.options.max_lio_size / self.options.block_size max_num_blocks = min(max_lio_blocks, self.options.blocks - block_pos) else: max_num_blocks = min(self.options.max_num_blocks, self.options.blocks - block_pos) num_blocks = random.randint(1, max_num_blocks) size = num_blocks * self.options.block_size area = (pos, pos+size) if (self._write_locks.search(area) or self._read_locks.search(area)): # Someone is currently writing to this position. self.log(2, 'writer skipping lock %r', area) else: break self._write_locks.insert(area) return pos, size, area
def worker(sleep): global l print 'working starting' l.acquire() coro.sleep_relative(sleep) print 'worker done, releasing' l.release()
def get_connected (self): if self.state == self.UNCONNECTED: self.state = self.CONNECTING for i in range (self._num_retries): try: coro.with_timeout (self._retry_timeout, self._connect) except coro.TimeoutError: coro.sleep_relative (self._retry_timeout) except OSError as e: # wait a bit, maybe it'll come back up... coro.sleep_relative (self._retry_timeout) except: self.state = self.UNCONNECTED raise else: return # ok, we give up! # fail any pending requests self.state = self.UNCONNECTED for thread in self.pending_requests.values(): # avoid a race: if the thread is already scheduled, then # it's about to be awakened with the result it was looking # for, so don't bother trying to interrupt it - this will # cause the latent interrupt to show up at a later and # bizarre point in the code. [see bug 5322 and others] if not thread.scheduled: thread.raise_exception (RPC_Server_Unreachable) # fail anyone waiting on connect self.connection_cv.raise_all (RPC_Server_Unreachable)
def start (self, addr, retries=5): """Start the web server listening on addr in a new coroutine. Try up to <retries> time to bind to that address. Raises an exception if the bind fails.""" self.addr = addr self.sock = self.create_sock() self.sock.set_reuse_addr() done = 0 save_errno = 0 while not done: for x in xrange (retries): try: self.sock.bind (addr) except OSError, why: if why.errno not in (errno.EADDRNOTAVAIL, errno.EADDRINUSE): raise else: save_errno = 0 if why.errno == errno.EADDRINUSE: was_eaddrinuse = 1 else: done = 1 break coro.sleep_relative (1) else: self.log ('cannot bind to %s:%d after 5 attempts, errno = %d' % (addr[0], addr[1], save_errno)) coro.sleep_relative (15)
def go (self): b = backdoor (self, line_separator='\n') # this is to avoid getting the banner/copyright/etc mixed in with ssh client pty/x11 warnings coro.sleep_relative (0.1) b.read_eval_print_loop() self.close() coro.print_stderr ('closed ssh backdoor from: %r\n' % (self.transport.transport.peer,))
def stun (self): self.stunned = True for pos in self.tail: self.arena.draw (pos, '*') coro.sleep_relative (5) self.stunned = False return not self.exit
def rate_thread(): n0 = 0 while 1: coro.sleep_relative (10) n1 = total_bytes delta = n1 - n0 n0 = n1 WR ('(%.2f)' % (delta / 10485760.))
def thingy (i): n = 0 while 1: coro.sleep_relative (5) LOG ('thingy loop', i, n) if i == 1: set_trace() n += 1
def popper (n): global live W ('>%d>' % (n,)) while True: x = the_fifo.pop() W ('[%d.%d]' % (n, x)) coro.sleep_relative (0) W ('<%d<' % (n,))
def test_1_trigger_signal (self): # XXX on osx, this sleep is necessary, otherwise this # too-quick SIGUSR1 is lost. coro.sleep_relative (0.1) pid = os.getpid() os.kill (pid, signal.SIGUSR1) coro.sleep_relative (0.1) self.assertEqual (signal_caught_flag, True)
def test_sleep_interrupt(): coro.spawn(_test_sleep_interrupt, coro.current()) # The other thread will interrupt me coro.print_stderr('There should be no pause here:\n') try: coro.sleep_relative(10) except coro.Interrupted, why: if why!='foobar': raise ValueError, 'Incorrect interrupt value.'
def tripwire(sleep_val): thread = coro.spawn(simple_thread) # let simple_thread run for a while: coro.sleep_relative(sleep_val) # use these values to try to trigger refcount bugs: thread.interrupt([]) thread.interrupt({}) thread.interrupt({'': []}) thread.interrupt([{}])
def tripwire(sleep_val): thread = coro.spawn (simple_thread) # let simple_thread run for a while: coro.sleep_relative (sleep_val) # use these values to try to trigger refcount bugs: thread.interrupt([]) thread.interrupt({}) thread.interrupt({'': []}) thread.interrupt([{}])
def test_sleep(): coro.print_stderr(' This pause should be 1 second.\n') coro.sleep_relative(1) coro.print_stderr(' This pause should be 1 second.\n') coro.sleep_relative(1.0) coro.print_stderr(' This pause should be 1 second.\n') coro.sleep_relative(1L * coro.ticks_per_sec) coro.print_stderr(' This pause should be 1 second.\n') coro.sleep_absolute(coro.now + 1L * coro.ticks_per_sec)
def test_sleep_interrupt(): coro.spawn(_test_sleep_interrupt, coro.current()) # The other thread will interrupt me coro.print_stderr('There should be no pause here:\n') try: coro.sleep_relative(10) except coro.Interrupted, why: if why != 'foobar': raise ValueError('Incorrect interrupt value.')
def new_connection_thread(): # give the servers time to start up and set addresses coro.sleep_relative (2) while 1: G.out_conn_sem.acquire (1) addr1 = new_random_addr() addr0 = get_my_addr (addr1) Connection (addr0, addr1) # avoid hammering the internets coro.sleep_relative (1)
def new_connection_thread(): # give the servers time to start up and set addresses coro.sleep_relative(2) while 1: addr1 = new_random_addr() if addr1 is not None: G.out_conn_sem.acquire(1) addr0 = get_my_addr(addr1) Connection(addr0, addr1) # avoid hammering the internets coro.sleep_relative(1)
def popper(): global finished, q, size for x in xrange(size): while True: data = q.dequeue() if data is not None: break if finished: break coro.sleep_relative(0) coro.sleep_relative(0)
def popper(): global finished, q, size for x in xrange(size): while 1: data = q.dequeue() if data is not None: break if finished: break coro.sleep_relative(0) coro.sleep_relative(0)
def login(self): self.send('Username: '******'Password: '******'Sorry, Charlie\n') self.sock.conn.close() raise NotAuthorized(u)
def login (self): self.send ('Username: '******'Password: '******'Sorry, Charlie\n') self.sock.conn.close() raise NotAuthorized (u)
def monitor_thread(): while 1: if gc.garbage: coro.print_stderr( "Warning: possible memory leak: len(gc.garbage)=%d\n" % (len(gc.garbage), )) coro.print_stderr("\tFirst %d objects in gc.garbage:\n" % (len(gc.garbage[:5]))) for x in gc.garbage[:5]: coro.print_stderr("\t %s\n" % (safe_repr(x), )) coro.sleep_relative(monitor_sleep_interval)
def test_with_timeout (self): def go(): print "timer" with self.assertRaises(coro.TimeoutError): # coro.with_timeout(2, coro.sleep_relative, 4) coro.with_timeout(2, coro.waitpid, os.getpid()) print "foo" coro.spawn(go) for i in range(5): coro.yield_slice() coro.sleep_relative(1)
def monitor_thread(): while True: if gc.garbage: coro.print_stderr ( "Warning: possible memory leak: len(gc.garbage)=%d\n" % (len(gc.garbage),) ) coro.print_stderr ( "\tFirst %d objects in gc.garbage:\n" % (len(gc.garbage[:5])) ) for x in gc.garbage[:5]: coro.print_stderr ("\t %s\n" % (safe_repr (x),)) coro.sleep_relative (monitor_sleep_interval)
def test_with_timeout(self): def go(): print "timer" with self.assertRaises(coro.TimeoutError): # coro.with_timeout(2, coro.sleep_relative, 4) coro.with_timeout(2, coro.waitpid, os.getpid()) print "foo" coro.spawn(go) for i in range(5): coro.yield_slice() coro.sleep_relative(1)
def test_with_timeout(self): def serve (port): s = coro.tcp_sock() s.bind (('', port)) s.listen (5) with self.assertRaises(coro.TimeoutError): conn, addr = coro.with_timeout(1, s.accept) # do this a second time to make sure no SimultaneousErrors occur with self.assertRaises(coro.TimeoutError): conn, addr = coro.with_timeout(1, s.accept) coro.spawn(serve, 8100) coro.yield_slice() coro.sleep_relative(3)
def go (self): while not self.exit: coro.sleep_relative (self.speed / 10000.0) if random.randrange (0, 20) == 10: if not self.turn(): return else: nx, ny = self.update() while self.arena[(nx, ny)] != ' ': if not self.turn(): return nx, ny = self.update() self.move ((nx, ny))
def session (conn, addr): conn.ssl.set_fd (conn.fd) try: print 'conn=', conn, conn.__class__ while 1: block = conn.recv (1000) if not block: break else: conn.send (block) finally: coro.sleep_relative (1000) W ('why for I exit?\n')
def test_with_timeout(self): def serve(port): s = coro.tcp_sock() s.bind(("", port)) s.listen(5) with self.assertRaises(coro.TimeoutError): conn, addr = coro.with_timeout(1, s.accept) # do this a second time to make sure no SimultaneousErrors occur with self.assertRaises(coro.TimeoutError): conn, addr = coro.with_timeout(1, s.accept) coro.spawn(serve, 8100) coro.yield_slice() coro.sleep_relative(3)
def thread_manager(self): """thread_manager(self) -> None This is the deadlock detection thread. """ while 1: try: coro.sleep_relative(self.deadlock_check_period) self.check_for_deadlocks() except coro.Interrupted: # Exiting. self.thread = None return except: coro.print_stderr(tb.traceback_string() + '\n')
def pusher(): global finished, q, size total = size while True: if total == 1: to_do = 1 else: to_do = random.randint(1, total / 2) total -= to_do for x in xrange(to_do): q.enqueue('some_data') if total == 0: break coro.sleep_relative(0) finished = 1
def test2(): coro.sleep_relative (5) f = ftp_client() coro.print_stderr ("connecting...\n") f.connect ('10.1.1.209') coro.print_stderr ("logging in...\n") f.cmd_user ('ftpguy') f.cmd_pass ('ftpguy') f.cmd_type ('I') coro.print_stderr ("sending file...\n") file = open ('ftp_client.py', 'rb') thunk = (lambda f=file: f.read (8192)) f.cmd_stor ('a_file.txt', thunk) coro.print_stderr ("done.\n") f.cmd_quit()
def test1(): coro.print_stderr ("waiting 5 seconds...\n") coro.sleep_relative (5) f = ftp_client() coro.print_stderr ("connecting...\n") f.connect ('squirl.nightmare.com') coro.print_stderr ("logging in...\n") f.cmd_user ('anonymous') f.cmd_pass ('rushing@') blocks = [] coro.print_stderr ("retrieving directory listing...\n") f.cmd_list (blocks.append) coro.print_stderr ("done.\n") f.cmd_quit() coro.print_stderr ('-'*20 + '\n') coro.print_stderr (''.join (blocks)) coro.print_stderr ('-'*20 + '\n')
def _reader(self, reader_num): selfish_acts = 1 self.reader_status[reader_num] = 'Starting.' try: while 1: if coro.get_now() > self._start_time + self.options.duration*coro.ticks_per_sec: self.reader_status[reader_num] = 'Finished.' return if not selfish_acts % self.options.greediness: self.reader_status[reader_num] = 'Greediness sleep.' coro.sleep_relative(0) if not self._have_blocks_to_read(): self.reader_status[reader_num] = 'Sleeping, waiting for writers to catch up.' coro.sleep_relative(0.1) continue pos, size, area = self._get_to_read() self.log(3, '%i: read(%i, %i)', reader_num, size, pos) try: if random.random() < self.options.cancel_percent/100.0: try: self.reader_status[reader_num] = 'Reading with cancel.' data = coro.with_timeout(0, coro.aio_read, self._fd, size, long(pos)) except coro.TimeoutError: self._read_cancel_success += 1 else: self._read_cancel_fail += 1 else: self.reader_status[reader_num] = 'Reading.' data = coro.aio_read(self._fd, size, long(pos)) expected_data = t_aio.make_data(pos, size) if data != expected_data: self._assertion_errors += 1 self.log(0, 'ERROR: data read=%i expected=%i pos=%i', len(data), size, pos) fname = tempfile.mktemp() f = open(fname, 'w') f.write(data) f.close() self.log(0, 'Wrote temp file %s', fname) finally: self._read_locks.delete(area) selfish_acts += 1 self._reads_finished += 1 self._bytes_read += size finally: self._worker_semaphore.release()
def _run (self): """Listens on the FTP port accepting connections and spawning sessions.""" self.thread_id = coro.current().thread_id() s = coro.make_socket (socket.AF_INET, socket.SOCK_STREAM) try: s.set_reuse_addr() done = 0 while not done: for x in xrange (5): try: was_eaddrinuse = 0 s.bind ((self.ip, self.port)) except OSError, why: if why[0] == errno.EACCES: coro.print_stderr( 'Access denied binding to %s:%i. Are you running as root?\n' % (self.ip, self.port)) return elif why[0] == errno.EADDRINUSE: was_eaddrinuse = 1 elif why[0] != errno.EADDRNOTAVAIL: raise else: done = 1 break coro.sleep_relative (1) else: coro.print_stderr ("cannot bind to %s:%d after 5 attempts\n" % (self.ip, self.port)) if was_eaddrinuse: qlog.write('FTPD.PORT_IN_USE', self.ip, str(self.port)) coro.sleep_relative (15) s.listen (1024) while 1: conn_list = s.accept_many() for conn, addr in conn_list: qlog.write('FTPD.CONNECTION', self.session_id, addr[0], self.ip) session = self.channel (self, conn, addr, self.session_id) self.session_id += 1 thread = coro.spawn (session.run) thread.set_name ( "%s_%d" % ( session.__class__.__name__, thread.thread_id() ) ) self.clients.append(session)
def test_multiple_sleep_interrupt(): # The first interrupt will work just fine coro.spawn(_test_multiple_sleep_interrupt_ok, coro.current()) # Since it has already been interrupted, you can't interrupt it ever again. # The following threads will fail. This is to be expected. # Someday we need to redesign how interrupts work so that these won't fail. # But that will be very tricky. coro.spawn(_test_multiple_sleep_interrupt_fail, coro.current()) coro.spawn(_test_multiple_sleep_interrupt_fail, coro.current()) coro.spawn(_test_multiple_sleep_interrupt_fail, coro.current()) # The other thread will interrupt me coro.print_stderr(' There should be no pause here:\n') try: coro.sleep_relative(10) except coro.Interrupted, why: if why != 'foobar': raise ValueError('Incorrect interrupt value.')
def run(self): self.thread_id = coro.current().thread_id() while not self.shutdown_flag: try: conn, addr = self.accept() client = self.create_connection(conn, addr) c = coro.spawn(client.run) c.set_name(('%s connection on %r' % ( self.__class__.__name__, addr, )).encode()) except coro.Shutdown: break except: LOG.exc() coro.sleep_relative(0.25) continue self.sock.close()
def _writer(self, writer_num): selfish_acts = 1 self._num_live_writers += 1 self.writer_status[writer_num] = 'Starting.' try: while 1: if coro.get_now() > self._start_time + self.options.duration*coro.ticks_per_sec: self.writer_status[writer_num] = 'Finished.' return if not selfish_acts % self.options.greediness: self.writer_status[writer_num] = 'Greediness sleep.' coro.sleep_relative(0) self.writer_status[writer_num] = 'Getting area to write.' pos, size, area = self._get_to_write() self.log(3, '%i: write(%i, %i)', writer_num, size, pos) data = t_aio.make_data(pos, size) try: if random.random() < self.options.cancel_percent/100.0: try: self.writer_status[writer_num] = 'Writing with cancel.' num_written = coro.with_timeout(0, coro.aio_write, self._fd, data, long(pos)) except coro.TimeoutError: self._write_cancel_success += 1 else: self._written.append((pos, size)) self._write_cancel_fail += 1 else: self.writer_status[writer_num] = 'Writing.' num_written = coro.aio_write(self._fd, data, long(pos)) if num_written != size: self.log(0, 'ERROR: Failed to write %i bytes (%i written).' % (size, num_written)) self._assertion_errors += 1 else: self._written.append((pos, size)) finally: self._write_locks.delete(area) selfish_acts += 1 #print len(self._written) self._writes_finished += 1 self._bytes_written += size finally: self._worker_semaphore.release() self._num_live_writers -= 1
def simple_thread(): global exitval, msg count = 0 try: while count < 10: #print "count is %d" % (count,) coro.sleep_relative(1.0) count += 1 msg = "thread was never interrupted" exitval = 1 except coro.Interrupted, why: if why == []: msg = "multiple interrupts policy succeeded" exitval = 0 elif why == [{}]: msg = "multiple interrupts policy reversed - it should use the first interrupt's value" exitval = 1 else: msg = "interrupts out of order: interrupt value is %r" % (why, ) exitval = 1
def _get_to_read(self): attempts = 0 while 1: # Pick a random location that has been written. # Don't pick anything that overlaps with stuff being written. attempts += 1 if not attempts % 3: self.log(2, 'Failed to pick read location 3 times.') coro.sleep_relative(0) block_index = random.randint(0, len(self._written)-1) pos, size = self._written[block_index] area = (pos, pos+size) if self._write_locks.search(area): self.log(2, 'reader skipping write lock %r', area) else: self._written.pop(block_index) break self._read_locks.insert(area) return pos, size, area
def _run(self, server_s): secure = self.is_secure() self.thread_id = coro.current().thread_id() while not self.shutdown_flag: try: conn, addr = server_s.accept() client = http_client() coro.spawn(client.run, conn, addr, self, self._handlers) except coro.Shutdown: # server-shutdown break except: qlog.write( 'COMMON.APP_FAILURE', ('%s accept handler error %s' % (self.__class__.__name__, coro.compact_traceback()))) coro.sleep_relative(0.25) continue server_s.close() return None
def wanderer(self): # spawn me! x = random.randint(100, self.w - 100) y = random.randint(100, self.h - 100) c = random.choice(colors) ob = circle(c, (x, y), 25) self.new_ob(ob) while 1: # pick a random direction heading = random.randint(0, 360) * (math.pi / 180.0) speed = (random.random() * 10) + 3 dx = math.cos(heading) * speed dy = math.sin(heading) * speed # go that way for 20 steps for i in range(20): self.move_ob(ob, dx, dy) # not working yet... # if not ob.range_check (0, 0, self.w, self.h): # W ('%r hit an edge!\n' % (ob,)) # dx = - (dx * 5) # dy = - (dy * 5) # self.move_ob (ob, dx, dy) coro.sleep_relative(self.sleep)