def process(path, file, hddsem): with open(path) as f: links = container[path.ext](f.read()) core.add_links(links) if config.delete_after_process is not None: delete = config.delete_after_process else: result = dict() delete = False elements = [ input.Text('Added links of container {}.\nDelete container now?'.format(os.path.basename(path))), input.Input('remember', 'checkbox', label='Remember decision?'), input.Choice('delete', choices=[ {"value": True, "content": "Yes"}, {"value": False, "content": "No"}, ])] try: result = input.get(elements, type='delete_container', parent=file, timeout=120) delete = result['delete'] except input.InputTimeout: log.warning('input timed out') except input.InputError: log.exception('input was aborted') if result.get('remember', False): config.delete_after_process = delete if delete: if file: file.erase_after_greenlet() else: os.remove(path) raise gevent.GreenletExit()
def __url_crawler__(self, url, idx): """Worker to crawl a url. Downloads the url, parses it and stores any new outgoing links. #TODO add a seperate pipeline for postprocessing.""" LOGGER = get_logger("__url_crawler__") LOGGER.debug("worker {idx}: starting to crawl {url}".format(url=url, idx=idx)) try: req = requests.get(url) LOGGER.debug("worker {idx}: download_success {url}".format( url=url, idx=idx)) except Exception as e: LOGGER.warn("worker {idx}: download_failed {url}".format(url=url, idx=idx)) req = None pass if req and req.status_code == 200: try: soup = BeautifulSoup(req.text, "html.parser") LOGGER.debug("worker {idx}: parsed {url}".format(url=url, idx=idx)) anchors = soup.find_all('a', href=True) LOGGER.debug("worker {idx}: found {anchors}".format( idx=idx, anchors=len(anchors))) anchors = [ self.__add_url_to_q__(anchor['href']) for anchor in anchors ] LOGGER.debug("worker {idx}: added {anchors}".format( idx=idx, anchors=sum(anchors))) except Exception as e: LOGGER.warn("adding urls failed") LOGGER.debug("exiting worker {idx}".format(idx=idx)) self.url_crawled.set() raise gevent.GreenletExit('success')
def worker(url): html = (D.post if post else D.get)(url, **kwargs) if cb: for url in (cb(D, url, html) or []): inq.put(url) worker_finished.set() raise gevent.GreenletExit('success')
def send(self, msg): """Wrapper for self.io.send, to detect any sending problem and call the destroy callbacks""" if not self.connected: self.kill() self.debug("not connected on send(): exiting greenlet") raise gevent.GreenletExit() self.io.send(msg)
def wrapped(ctx, *args, **kwargs): io = ctx.io if not io.session.connected: ctx.kill() ctx.debug("not connected on %s: exiting greenlet", fn.__name__) raise gevent.GreenletExit() return fn(ctx, *args, **kwargs)
def test_together_kill(self): sock = mock.Mock() with self.assertRaises(gevent.GreenletExit): with utils.SocketCloser(sock): sock.make_call() raise gevent.GreenletExit() sock.make_call.assert_called_once_with() sock.close.assert_called_once_with()
def test_exit_kill_thresh(self): sock = mock.Mock() closer = utils.SocketCloser(sock, 10) result = closer.__exit__(gevent.GreenletExit, gevent.GreenletExit(), []) self.assertEqual(result, None) sock.close.assert_called_once_with() self.assertEqual(closer.errors, 0)
def fatal(self, msg, abort_greenlet=True): if self.input: interface.call('input', 'abort', id=self.input.id) self.input = None with transaction: self.last_error = msg self.enabled = False self.log.error(msg) event.fire('file:fatal_error', self) if abort_greenlet: raise gevent.GreenletExit()
def run(self): self.setup_in_process() last_clean = datetime.utcnow() while True: # Run forever and ever # Clean out visited every 5 minutes if datetime.utcnow() - last_clean > timedelta(minutes=5): self._log.debug("Cleaning cache...") self.__cache.clean_and_save() last_clean = datetime.utcnow() try: # Get next object to process event = self.__queue.get(block=True, timeout=5) except gevent.queue.Empty: # Check if the process should exit process if self.__event.is_set(): break # Explict context yield gevent.sleep(0) continue try: kind = type(event) self._log.debug("Processing event: %s", event.id) if kind == Events.MonEvent: self.process_monster(event) elif kind == Events.StopEvent: self.process_stop(event) elif kind == Events.GruntEvent: self.process_grunt(event) elif kind == Events.GymEvent: self.process_gym(event) elif kind == Events.EggEvent: self.process_egg(event) elif kind == Events.RaidEvent: self.process_raid(event) elif kind == Events.WeatherEvent: self.process_weather(event) elif kind == Events.QuestEvent: self.process_quest(event) else: self._log.error( "!!! Manager does not support {} events!".format(kind)) self._log.debug("Finished event: %s", event.id) except Exception as e: self._log.error("Encountered error during processing: " "{}: {}".format(type(e).__name__, e)) self._log.error("Stack trace: \n {}" "".format(traceback.format_exc())) # Explict context yield gevent.sleep(0) # Save cache and exit self.__cache.clean_and_save() raise gevent.GreenletExit()
def test_together_kill_thresh(self): sock = mock.Mock() closer = utils.SocketCloser(sock, 10) with self.assertRaises(gevent.GreenletExit): with closer: sock.make_call() raise gevent.GreenletExit() sock.make_call.assert_called_once_with() sock.close.assert_called_once_with() self.assertEqual(closer.errors, 0)
def retry(self, msg, seconds, need_reconnect=False): seconds = float(seconds) self.log.info(u'retry in {} seconds: {}; reconnect: {}'.format( seconds, msg, need_reconnect)) with transaction: self.last_error = msg self.need_reconnect = need_reconnect self.next_try = time.time() + seconds if self.input: interface.call('input', 'abort', id=self.input.id) self.input = None raise gevent.GreenletExit()
def retry( self, msg, seconds, _=False ): # _ is a placeholder to make ErrorFunctions work. we have no need_reconnect with transaction: self.next_try = gevent.spawn_later(seconds, self.reset_retry) self.next_try.eta = time.time() + seconds self.last_error = msg self.log.info('retry in {} seconds: {}'.format(seconds, msg)) raise gevent.GreenletExit()
def retry(self, msg, seconds, need_reconnect=False): seconds = float(seconds) if self.input: interface.call('input', 'abort', id=self.input.id) self.input = None with transaction: self.last_error = msg self.need_reconnect = need_reconnect self.next_try = gevent.spawn_later(seconds, self.reset_retry) self.next_try.eta = time.time() + seconds self.retry_num += 1 self.log.info(u'retry in {} seconds: {}; reconnect: {}'.format( seconds, msg, need_reconnect)) event.fire('file:retry', self) raise gevent.GreenletExit()
def add_torrent(t, file=None): """t = Torrent object instance or string with torrent data""" if isinstance(t, str): try: t = Torrent(data=t, options=dict(save_path=core.config.download_dir, auto_managed=False)) except RuntimeError: if file is not None: file.fatal('torrent already exists') else: log.warning('torrent already exists') raise gevent.GreenletExit() try: name = t.name for package in core.packages(): if package.system == 'torrent' and package.name == name: return links = list() #rename = list() for i, f in enumerate(t.files): fname = convert_torrent_name(name, f['path']) #if fname != f['path']: # rename.append((i, fname)) link = {} link['url'] = 'torrent:///?{}#{}'.format(urllib.quote(t.id), i) link['name'] = fname link['size'] = f['size'] link['state'] = 'collect' links.append(link) #if rename: # t.rename_files(rename) t.save_torrent() ids = core.add_links(links, package_name=name, package_id=t.id, system='torrent') for file in core.files(): if file.id in ids: event.fire('file:checked', file) finally: t.remove()
def _upload(thread, queue, errors): if verbose: print 'Thread %s: start' % thread while True: try: file = queue.get_nowait() except gevent.queue.Empty: if verbose: if verbose > 1: print 'Thread %3s: queue empty' % thread print 'Thread %3s: exiting' % thread raise gevent.GreenletExit() else: with open(file['path'], 'rb') as f: body = f.read() if verbose > 1: print 'Thread %3s: uploading %s' % (thread, file['name']) try: r = s.put('%s/%s/%s' % (self.endpoint, container, file['name']), data=body, headers={ 'X-Auth-Token': self.token }) except: e = sys.exc_info()[1] errors.append({ 'name': file['name'], 'container': container, 'exception': str(e) }) else: if r.status_code == 401: raise AuthenticationError if r.status_code != 201: errors.append({ 'name': file['name'], 'container': container, 'status_code': r.status_code, 'headers': r.headers, 'response': json.loads(r.text) }) finally: if verbose > 1: print ('Thread %3s: upload complete for %s' % (thread, file['name']))
def test_thread_error_greenletexit(self, mock_closed, mock_close): thread = mock.Mock(**{ 'successful.return_value': False, 'exception': gevent.GreenletExit(), }) tend = tcp.TCPTendril('manager', self.sock) tend._send_thread = mock.Mock() tend._recv_thread = mock.Mock() tend._thread_error(thread) mock_close.assert_called_once_with() self.assertFalse(mock_closed.called) # Ensure we didn't overwrite the threads self.assertNotEqual(tend._send_thread, None) self.assertNotEqual(tend._recv_thread, None)
def _recv(self): """ Implementation of the receive thread. Waits for data to arrive on the socket, then passes the data through the defined receive framer and sends it on to the application. """ # Outer loop: receive some data while True: # Wait until we can go self._recv_lock.release() gevent.sleep() # Yield to another thread self._recv_lock.acquire() recv_buf = self._sock.recv(self.recv_bufsize) # If it's empty, the peer closed the other end if not recv_buf: # Manually kill the send thread; do this manually # instead of calling close() because close() will kill # us, and since close() would be running in our thread # context, it would never get around to killing the # send thread if self._send_thread: self._send_thread.kill() self._send_thread = None # Manually close the socket self._sock.close() self._sock = None # Make sure the manager knows we're closed super(TCPTendril, self).close() # Notify the application self.closed() # As our last step, commit seppuku; this will keep # _thread_error() from notifying the application of an # erroneous exit from the receive thread raise gevent.GreenletExit() # Process the received data self._recv_frameify(recv_buf)
def ask_buy_premium_dialog(hoster, url): with premium_lock: if hoster in config.ignore_ask_premium: return try: elements = [ input.Text("Buy premium account for hoster {}?".format(hoster)), input.Choice('answer', choices=[ {"value": 'yes', "content": "Yes", "link": url}, {"value": 'no', "content": "No"}, {"value": 'already', "content": "I already have an account"}, {"value": 'never', "content": "No and never ask again"} ]), ] result = input.get(elements) if result['answer'] == 'already': elements = [ input.Text('Please enter your account details for {}'.format(hoster)), input.Float('left'), [input.Text('Username:'******'username')], [input.Text('Password:'******'password', 'password')], input.Float('center'), input.Text(''), input.Choice('answer', choices=[ {"value": 'add', "content": "Add Account", 'ok': True}, {"value": 'cancel', "content": "Cancel", 'cancel': True} ]) ] result = input.get(elements, close_aborts=True) if result['answer'] == 'add' and result['username'] and result['password']: interface.call('account', 'add', name=hoster, username=result['username'], password=result['password']) raise gevent.GreenletExit() elif result['answer'] == 'never': if hoster not in config.ignore_ask_premium: with transaction: config.ignore_ask_premium.append(hoster) except KeyError: pass except input.InputTimeout: log.warning('input timed out') except input.InputError: log.error('input was aborted')
class TestUDPTendrilManager(unittest.TestCase): @mock.patch.object(manager.TendrilManager, '__init__') def test_init(self, mock_init): manager = udp.UDPTendrilManager() mock_init.assert_called_once_with(None) self.assertEqual(manager._sock, None) self.assertIsInstance(manager._sock_event, event.Event) self.assertEqual(manager._sock_event.is_set(), False) @mock.patch.object(manager.TendrilManager, 'start') def test_start(self, mock_start): manager = udp.UDPTendrilManager() manager._sock = 'sock' manager._sock_event = mock.Mock() manager.start('acceptor', 'wrapper') mock_start.assert_called_once_with('acceptor', 'wrapper') self.assertEqual(manager._sock, None) manager._sock_event.clear.assert_called_once_with() @mock.patch.object(manager.TendrilManager, 'stop') def test_stop(self, mock_stop): manager = udp.UDPTendrilManager() manager._sock = 'sock' manager._sock_event = mock.Mock() manager.stop('thread') mock_stop.assert_called_once_with('thread') self.assertEqual(manager._sock, None) manager._sock_event.clear.assert_called_once_with() @mock.patch.object(manager.TendrilManager, 'shutdown') def test_shutdown(self, mock_shutdown): manager = udp.UDPTendrilManager() manager._sock = 'sock' manager._sock_event = mock.Mock() manager.shutdown() mock_shutdown.assert_called_once_with() self.assertEqual(manager._sock, None) manager._sock_event.clear.assert_called_once_with() @mock.patch.object(manager.TendrilManager, 'connect') @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock()) @mock.patch.object(udp.UDPTendrilManager, 'local_addr', ('0.0.0.0', 8880)) def test_connect(self, mock_UDPTendril, mock_track_tendril, mock_connect): acceptor = mock.Mock() manager = udp.UDPTendrilManager() tend = manager.connect(('127.0.0.1', 8080), acceptor) mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor, None) mock_UDPTendril.assert_called_once_with(manager, ('0.0.0.0', 8880), ('127.0.0.1', 8080)) acceptor.assert_called_once_with(mock_UDPTendril.return_value) mock_track_tendril.assert_called_once_with( mock_UDPTendril.return_value) self.assertEqual(id(tend), id(mock_UDPTendril.return_value)) @mock.patch.object(manager.TendrilManager, 'connect') @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock()) @mock.patch.object(udp.UDPTendrilManager, 'local_addr', ('0.0.0.0', 8880)) def test_connect_rejected(self, mock_UDPTendril, mock_track_tendril, mock_connect): acceptor = mock.Mock(side_effect=application.RejectConnection()) manager = udp.UDPTendrilManager() tend = manager.connect(('127.0.0.1', 8080), acceptor) mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor, None) mock_UDPTendril.assert_called_once_with(manager, ('0.0.0.0', 8880), ('127.0.0.1', 8080)) acceptor.assert_called_once_with(mock_UDPTendril.return_value) self.assertFalse(mock_track_tendril.called) self.assertEqual(tend, None) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'recvfrom.side_effect': TestException(), 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock()) def test_listener_creator(self, mock_UDPTendril, mock_track_tendril, mock_socket): acceptor = mock.Mock() manager = udp.UDPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_UDPTendril.called) self.assertFalse(acceptor.called) self.assertFalse(mock_track_tendril.called) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'recvfrom.side_effect': TestException(), 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock()) @mock.patch.object(udp.UDPTendrilManager, 'recv_bufsize', 8192) def test_listener_creator_recv_bufsize(self, mock_UDPTendril, mock_track_tendril, mock_socket): acceptor = mock.Mock() manager = udp.UDPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.recvfrom(8192), mock.call.recvfrom(8192), mock.call.recvfrom(8192), mock.call.recvfrom(8192), mock.call.recvfrom(8192), mock.call.recvfrom(8192), mock.call.recvfrom(8192), mock.call.recvfrom(8192), mock.call.recvfrom(8192), mock.call.recvfrom(8192), mock.call.recvfrom(8192), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_UDPTendril.called) self.assertFalse(acceptor.called) self.assertFalse(mock_track_tendril.called) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'recvfrom.side_effect': gevent.GreenletExit(), 'getsockname.return_value': ('127.0.0.1', 8080), 'close.side_effect': socket.error(), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock()) def test_listener_killed(self, mock_UDPTendril, mock_track_tendril, mock_socket): acceptor = mock.Mock() manager = udp.UDPTendrilManager() manager.running = True with self.assertRaises(gevent.GreenletExit): manager.listener(acceptor, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.recvfrom(4096), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_UDPTendril.called) self.assertFalse(acceptor.called) self.assertFalse(mock_track_tendril.called) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'recvfrom.side_effect': TestException(), 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock()) def test_listener_wrapper(self, mock_UDPTendril, mock_track_tendril, mock_socket): wrapped_sock = mock.Mock(**{ 'recvfrom.side_effect': TestException(), }) wrapper = mock.Mock(return_value=wrapped_sock) acceptor = mock.Mock() manager = udp.UDPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, wrapper) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.getsockname(), ]) wrapper.assert_called_once_with(mock_socket.return_value) wrapped_sock.assert_has_calls([ mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_UDPTendril.called) self.assertFalse(acceptor.called) self.assertFalse(mock_track_tendril.called) @mock.patch.object(socket, 'socket', return_value=mock.Mock(**{ 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock()) def test_listener_noacceptor(self, mock_UDPTendril, mock_track_tendril, mock_socket): msgs = ['msg1', 'msg2', 'msg3'] mock_socket.return_value.recvfrom.side_effect = [ (msgs[0], ('127.0.0.2', 8082)), (msgs[1], ('127.0.0.3', 8083)), (msgs[2], ('127.0.0.4', 8084)), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), ] tend = mock.Mock() manager = udp.UDPTendrilManager() manager.running = True manager.tendrils[(('127.0.0.1', 8080), ('127.0.0.3', 8083))] = tend with self.assertRaises(TestException): manager.listener(None, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_UDPTendril.called) self.assertFalse(mock_track_tendril.called) tend._recv_frameify.assert_called_once_with('msg2') @mock.patch.object(socket, 'socket', return_value=mock.Mock(**{ 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock()) def test_listener_recv_frameify_error(self, mock_UDPTendril, mock_track_tendril, mock_socket): mock_socket.return_value.recvfrom.side_effect = [ ('frame', ('127.0.0.3', 8083)), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), ] tend = mock.Mock(**{'_recv_frameify.side_effect': TestException()}) manager = udp.UDPTendrilManager() manager.running = True manager.tendrils[(('127.0.0.1', 8080), ('127.0.0.3', 8083))] = tend with self.assertRaises(TestException): manager.listener(None, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_UDPTendril.called) self.assertFalse(mock_track_tendril.called) tend._recv_frameify.assert_called_once_with('frame') tend.close.assert_called_once_with() self.assertEqual(tend.closed.call_count, 1) args = tend.closed.call_args[0] self.assertEqual(len(args), 1) self.assertIsInstance(args[0], TestException) @mock.patch.object(socket, 'socket', return_value=mock.Mock(**{ 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock()) def test_listener_acceptor(self, mock_UDPTendril, mock_track_tendril, mock_socket): msgs = ['msg1', 'msg2', 'msg3'] mock_socket.return_value.recvfrom.side_effect = [ (msgs[0], ('127.0.0.2', 8082)), (msgs[1], ('127.0.0.3', 8083)), (msgs[2], ('127.0.0.4', 8084)), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), ] tendrils = [mock.Mock(), mock.Mock(), mock.Mock()] mock_UDPTendril.side_effect = [tendrils[0], tendrils[2]] acceptor = mock.Mock() manager = udp.UDPTendrilManager() manager.running = True manager.tendrils[(('127.0.0.1', 8080), ('127.0.0.3', 8083))] = tendrils[1] with self.assertRaises(TestException): manager.listener(acceptor, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) mock_UDPTendril.assert_has_calls([ mock.call(manager, ('127.0.0.1', 8080), ('127.0.0.2', 8082)), mock.call(manager, ('127.0.0.1', 8080), ('127.0.0.4', 8084)), ]) acceptor.assert_has_calls([ mock.call(tendrils[0]), mock.call(tendrils[2]), ]) mock_track_tendril.assert_has_calls([ mock.call(tendrils[0]), mock.call(tendrils[2]), ]) tendrils[0]._recv_frameify.assert_called_once_with('msg1') tendrils[1]._recv_frameify.assert_called_once_with('msg2') tendrils[2]._recv_frameify.assert_called_once_with('msg3') @mock.patch.object(socket, 'socket', return_value=mock.Mock(**{ 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock()) def test_listener_rejector(self, mock_UDPTendril, mock_track_tendril, mock_socket): mock_socket.return_value.recvfrom.side_effect = [ ('msg1', ('127.0.0.2', 8082)), ('msg2', ('127.0.0.3', 8083)), ('msg3', ('127.0.0.4', 8084)), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), ] tendrils = [mock.Mock(), mock.Mock(), mock.Mock()] mock_UDPTendril.side_effect = tendrils[:] acceptor = mock.Mock(side_effect=application.RejectConnection()) manager = udp.UDPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.recvfrom(4096), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) mock_UDPTendril.assert_has_calls([ mock.call(manager, ('127.0.0.1', 8080), ('127.0.0.2', 8082)), mock.call(manager, ('127.0.0.1', 8080), ('127.0.0.3', 8083)), mock.call(manager, ('127.0.0.1', 8080), ('127.0.0.4', 8084)), ]) acceptor.assert_has_calls([ mock.call(tendrils[0]), mock.call(tendrils[1]), mock.call(tendrils[2]), ]) self.assertFalse(mock_track_tendril.called) self.assertFalse(tendrils[0]._recv_frameify.called) self.assertFalse(tendrils[1]._recv_frameify.called) self.assertFalse(tendrils[2]._recv_frameify.called) def test_sock_getter_notrunning(self): manager = udp.UDPTendrilManager() manager._sock = 'socket' manager._sock_event = mock.Mock() result = manager.sock self.assertEqual(result, None) self.assertFalse(manager._sock_event.wait.called) def test_sock_getter(self): manager = udp.UDPTendrilManager() manager.running = True manager._sock = 'socket' manager._sock_event = mock.Mock() result = manager.sock self.assertEqual(result, 'socket') manager._sock_event.wait.assert_called_once_with()
class Crawler(object): """Simple crawler based on gcrawler, but with a rolling inq that can be added to. The crawler is done when the workers have no more jobs and there are no more urls in the queue.""" seen_jobs = set() def __init__(self, spider, timeout=2, worker_count=4, pipeline_size=100): self.spider = spider self.spider.crawler = self self.timeout = timeout self.count = worker_count self.inq = queue.Queue(0) self.outq = queue.Queue(pipeline_size) self.jobq = queue.Queue() self.pool = pool.Pool(worker_count) self.worker_finished = event.Event() for job in getattr(self.spider, 'jobs', []): self.add_job(job) self.cache = '.cache' def start(self): """Start the crawler. Starts the scheduler and pipeline first, then adds jobs to the pool and waits for the scheduler and pipeline to finish. The scheduler itself shuts down the pool and waits on that, so it's not required to watch the pool.""" # add the spider's jobs first for url in self.spider.urls: self.add_job(url) # start the scheduler & the pipeline self.scheduler_greenlet = gevent.spawn(self.scheduler) self.pipeline_greenlet = gevent.spawn(self.pipeline) self.scheduler_greenlet.join() def add_job(self, job, **kwargs): """Add a job to the queue. The job argument can either be a Job object or a url with keyword arguments to be passed to the Job constructor.""" if isinstance(job, basestring): job = Job(job, **kwargs) # do not visit previously viewed urls if job in self.seen_jobs: return self.jobq.put(job) def scheduler(self): """A simple job scheduler. Presuming it's started after there is at least one job, it feeds jobs to the job queue to a synchronous worker job queue one at a time. When the worker queue fills up, the scheduler will block on the put() operation. When job queue is empty, the scheduler will wait for the workers to finish. If the job queue is empty and no workers are active, the pool's stopped.""" logger = logging.getLogger(__name__ + '.scheduler') while True: # join dead greenlets for greenlet in list(self.pool): if greenlet.dead: self.pool.discard(greenlet) try: logger.debug("Fetching job from job queue.") job = self.jobq.get_nowait() except queue.Empty: logger.debug("No jobs remaining.") if self.pool.free_count() != self.pool.size: logger.debug("%d workers remaining, waiting..." % (self.pool.size - self.pool.free_count())) self.worker_finished.wait() self.worker_finished.clear() continue else: logger.debug("No workers left, shutting down.") return self.shutdown() self.pool.spawn(self.worker, job) def shutdown(self): """Shutdown the crawler after the pool has finished.""" self.pool.join() self.outq.put(StopIteration) self.pipeline_greenlet.join() return True def worker(self, job, logger=logging.getLogger(__name__ + '.worker')): """A simple worker that fetches urls based on jobs and puts the ammended jobs on the outq for processing in the pipeline thread. After each job is fetched, but before the worker sets the finished event, the spider's preprocess method will be called on the job. This is its opportunity to add urls to the job queue. Heavy processing should be done via the pipeline in postprocess.""" logger.debug("starting: %r" % job) # you need to create a new Http instance per greenlet, see ticket: # http://code.google.com/p/httplib2/issues/detail?id=5 h = httplib2.Http(self.cache) try: job.response, job.data = h.request(job.url, method=job.method, headers=job.headers) self.spider.preprocess(job) except Exception, e: logger.error("Preprocessing error:\n%s" % traceback.format_exc()) self.outq.put(job) self.worker_finished.set() logger.debug("finished: %r" % job) raise gevent.GreenletExit('success')
def fatal(self, msg): with transaction: self.last_error = msg self.enabled = False self.log.error(msg) raise gevent.GreenletExit()
def test_00_init(self, mock_SimpleConsumer, mock_KafkaClient, mock_UVEServer): self.test_spec = [ TestStage(i=PartHandlerInput( redis_instances=set([("127.0.0.1", 44444, 0)]), get_part={ "127.0.0.1:44444": { "gen1": { "ObjectXX:uve1": set(["type1"]) } } }, get_messages=[ OffsetAndMessage( offset=0, message=Message( magic=0, attributes=0, key='', value=( '{"message":"UVEUpdate","key":"ObjectYY:uve2",' '"type":"type2","gen":"gen1","coll":' '"127.0.0.1:44444","deleted":false}'))) ]), o=PartHandlerOutput(callbacks=[ { "ObjectXX:uve1": None }, { "ObjectYY:uve2": set(["type2"]) }, ], uvedb=None)), TestStage( i=PartHandlerInput(redis_instances=gevent.GreenletExit(), get_part=None, get_messages=None), o=PartHandlerOutput(callbacks=[ { "ObjectXX:uve1": None, "ObjectYY:uve2": None }, ], uvedb={ "127.0.0.1:44444": { "gen1": { "ObjectXX:uve1": set(["type1"]), "ObjectYY:uve2": set(["type2"]) } } }), ) ] mock_UVEServer.return_value.redis_instances.side_effect = \ [x.i.redis_instances for x in self.test_spec] mock_UVEServer.return_value.get_part.side_effect = \ [x.i.get_part for x in self.test_spec if x.i.get_part is not None] mock_SimpleConsumer.return_value.get_messages.side_effect = \ [x.i.get_messages for x in self.test_spec] self.ph = UveStreamProc('no-brokers', 1, "uve-1", logging, self.callback_proc, "127.0.0.1", mock_UVEServer.return_value) self.ph.start() res, db = self.ph.get(timeout=10) if (isinstance(res, AssertionError)): raise res self.assertEqual(db, self.test_spec[-1].o.uvedb)
def run_step(self, step_config): """Runs a step. Args: step_config (StepConfig): The step configuration to run. Returns: A StepData object containing the result of the finished step. """ # TODO(iannucci): When subannotations are handled with annotee, move # `allow_subannotations` into recipe_module/step. name_tokens = self._record_step_name(step_config.name) # TODO(iannucci): Start with had_exception=True and overwrite when we know # we DIDN'T have an exception. ret = StepData(name_tokens, ExecutionResult()) try: self._step_runner.register_step_config(name_tokens, step_config) except: # Test data functions are not allowed to raise exceptions. Instead of # letting user code catch these, we crash the test immediately. _log_crash(self._stream_engine, "register_step_config(%r)" % (ret.name, )) raise CrashEngine("Registering step_config failed for %r." % (ret.name)) step_stream = self._stream_engine.new_step_stream( name_tokens, step_config.allow_subannotations) debug_log = step_stream.new_log_stream('$debug') caught = None try: # If there's a parent step on the stack, add `ret` to its children. self._step_stack[-1].children_steps.append(ret) # initialize presentation to show an exception. ret.presentation = StepPresentation(step_config.name) ret.presentation.status = 'EXCEPTION' self._step_stack.append(_ActiveStep(ret, step_stream, False)) # _run_step should never raise an exception, except for GreenletExit caught = _run_step(debug_log, ret, step_stream, self._step_runner, self._resource, step_config, self._environ, self._start_dir) # TODO(iannucci): remove this trigger specs crap if step_config.trigger_specs: _trigger_builds(step_stream, step_config.trigger_specs) # NOTE: See the accompanying note in stream.py. step_stream.reset_subannotation_state() ret.finalize() # If there's a buffered exception, we raise it now. if caught: # TODO(iannucci): Python3 incompatible. raise caught[0], caught[1], caught[2] # If the step was cancelled, raise GreenletExit. if ret.exc_result.was_cancelled: raise gevent.GreenletExit() if ret.presentation.status == 'SUCCESS': return ret # Otherwise, we raise an appropriate error based on # ret.presentation.status exc = { 'FAILURE': recipe_api.StepFailure, 'WARNING': recipe_api.StepWarning, 'EXCEPTION': recipe_api.InfraFailure, }[ret.presentation.status] # TODO(iannucci): Use '|' instead of '.' raise exc('.'.join(name_tokens), ret) finally: # per sys.exc_info this is recommended in python 2.x to avoid creating # garbage cycles. del caught debug_log.close()
def spawn_to_kill() -> None: gevent.spawn(greenlet.throw, gevent.GreenletExit())
class TestTCPTendrilManager(unittest.TestCase): @mock.patch.object(socket, 'socket', return_value=mock.Mock()) @mock.patch.object(manager.TendrilManager, 'connect') @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_connect_basic(self, mock_TCPTendril, mock_track_tendril, mock_connect, mock_socket): acceptor = mock.Mock() manager = tcp.TCPTendrilManager() tend = manager.connect(('127.0.0.1', 8080), acceptor) mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.connect(('127.0.0.1', 8080)), ]) mock_TCPTendril.assert_called_once_with(manager, mock_socket.return_value) acceptor.assert_called_once_with(mock_TCPTendril.return_value) mock_track_tendril.assert_called_once_with( mock_TCPTendril.return_value) mock_TCPTendril.return_value._start.assert_called_once_with() self.assertEqual(id(tend), id(mock_TCPTendril.return_value)) @mock.patch.object(socket, 'socket', return_value=mock.Mock()) @mock.patch.object(manager.TendrilManager, 'connect') @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_connect_wrapper(self, mock_TCPTendril, mock_track_tendril, mock_connect, mock_socket): wrapped_sock = mock.Mock() wrapper = mock.Mock(return_value=wrapped_sock) acceptor = mock.Mock() manager = tcp.TCPTendrilManager(('127.0.0.2', 8880)) tend = manager.connect(('127.0.0.1', 8080), acceptor, wrapper) mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor, wrapper) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('127.0.0.2', 8880)), mock.call.connect(('127.0.0.1', 8080)), ]) wrapper.assert_called_once_with(mock_socket.return_value) mock_TCPTendril.assert_called_once_with(manager, wrapped_sock) acceptor.assert_called_once_with(mock_TCPTendril.return_value) mock_track_tendril.assert_called_once_with( mock_TCPTendril.return_value) mock_TCPTendril.return_value._start.assert_called_once_with() self.assertEqual(id(tend), id(mock_TCPTendril.return_value)) @mock.patch.object(socket, 'socket', return_value=mock.Mock()) @mock.patch.object(manager.TendrilManager, 'connect') @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_connect_failure(self, mock_TCPTendril, mock_track_tendril, mock_connect, mock_socket): acceptor = mock.Mock(side_effect=TestException()) manager = tcp.TCPTendrilManager() with self.assertRaises(TestException): tend = manager.connect(('127.0.0.1', 8080), acceptor) mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.connect(('127.0.0.1', 8080)), mock.call.close(), ]) mock_TCPTendril.assert_called_once_with(manager, mock_socket.return_value) acceptor.assert_called_once_with(mock_TCPTendril.return_value) self.assertFalse(mock_track_tendril.called) self.assertFalse(mock_TCPTendril.return_value._start.called) @mock.patch.object( socket, 'socket', return_value=mock.Mock(**{ 'close.side_effect': socket.error(), })) @mock.patch.object(manager.TendrilManager, 'connect') @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_connect_failure_closeerror(self, mock_TCPTendril, mock_track_tendril, mock_connect, mock_socket): acceptor = mock.Mock(side_effect=TestException()) manager = tcp.TCPTendrilManager() with self.assertRaises(TestException): tend = manager.connect(('127.0.0.1', 8080), acceptor) mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.connect(('127.0.0.1', 8080)), mock.call.close(), ]) mock_TCPTendril.assert_called_once_with(manager, mock_socket.return_value) acceptor.assert_called_once_with(mock_TCPTendril.return_value) self.assertFalse(mock_track_tendril.called) self.assertFalse(mock_TCPTendril.return_value._start.called) @mock.patch.object(socket, 'socket', return_value=mock.Mock()) @mock.patch.object(manager.TendrilManager, 'connect') @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_connect_reject_connection(self, mock_TCPTendril, mock_track_tendril, mock_connect, mock_socket): acceptor = mock.Mock(side_effect=application.RejectConnection) manager = tcp.TCPTendrilManager() tend = manager.connect(('127.0.0.1', 8080), acceptor) self.assertEqual(tend, None) mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor, None) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.bind(('', 0)), mock.call.connect(('127.0.0.1', 8080)), mock.call.close(), ]) mock_TCPTendril.assert_called_once_with(manager, mock_socket.return_value) acceptor.assert_called_once_with(mock_TCPTendril.return_value) self.assertFalse(mock_track_tendril.called) self.assertFalse(mock_TCPTendril.return_value._start.called) @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock()) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_listener_noacceptor(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): manager = tcp.TCPTendrilManager() with self.assertRaises(TestException): manager.listener(None, None) mock_sleep.assert_called_once_with(600) self.assertFalse(mock_socket.called) @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'accept.side_effect': TestException(), 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_listener_creator(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): acceptor = mock.Mock() manager = tcp.TCPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) self.assertFalse(mock_sleep.called) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.listen(1024), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_TCPTendril.called) self.assertFalse(acceptor.called) self.assertFalse(mock_track_tendril.called) @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'accept.side_effect': TestException(), 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) @mock.patch.object(tcp.TCPTendrilManager, 'backlog', 4096) def test_listener_creator_backlog(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): acceptor = mock.Mock() manager = tcp.TCPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) self.assertFalse(mock_sleep.called) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.listen(4096), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_TCPTendril.called) self.assertFalse(acceptor.called) self.assertFalse(mock_track_tendril.called) @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'accept.side_effect': gevent.GreenletExit(), 'getsockname.return_value': ('127.0.0.1', 8080), 'close.side_effect': socket.error(), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_listener_killed(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): acceptor = mock.Mock() manager = tcp.TCPTendrilManager() manager.running = True with self.assertRaises(gevent.GreenletExit): manager.listener(acceptor, None) self.assertFalse(mock_sleep.called) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.listen(1024), mock.call.accept(), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_TCPTendril.called) self.assertFalse(acceptor.called) self.assertFalse(mock_track_tendril.called) @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'accept.side_effect': TestException(), 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_listener_wrapper(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): wrapped_sock = mock.Mock(**{ 'accept.side_effect': TestException(), }) wrapper = mock.Mock(return_value=wrapped_sock) acceptor = mock.Mock() manager = tcp.TCPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, wrapper) self.assertFalse(mock_sleep.called) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), mock.call.bind(('', 0)), mock.call.getsockname(), ]) wrapper.assert_called_once_with(mock_socket.return_value) wrapped_sock.assert_has_calls([ mock.call.listen(1024), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_TCPTendril.called) self.assertFalse(acceptor.called) self.assertFalse(mock_track_tendril.called) @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'accept.side_effect': TestException(), 'getsockname.return_value': ('127.0.0.1', 8080), 'listen.side_effect': TestException(), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_listener_nolisten(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): acceptor = mock.Mock() manager = tcp.TCPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) self.assertFalse(mock_sleep.called) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.listen(1024), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_TCPTendril.called) self.assertFalse(acceptor.called) self.assertFalse(mock_track_tendril.called) @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock(**{ 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril') def test_listener_acceptor(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): clis = [mock.Mock(), mock.Mock(), mock.Mock()] mock_socket.return_value.accept.side_effect = [ (clis[0], ('127.0.0.2', 8082)), (clis[1], ('127.0.0.3', 8083)), (clis[2], ('127.0.0.4', 8084)), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), ] tendrils = [mock.Mock(), mock.Mock(), mock.Mock()] mock_TCPTendril.side_effect = tendrils[:] acceptor = mock.Mock(side_effect=[ mock.Mock(), TestException(), mock.Mock(), ]) manager = tcp.TCPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) self.assertFalse(mock_sleep.called) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.listen(1024), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) mock_TCPTendril.assert_has_calls([ mock.call(manager, clis[0], ('127.0.0.2', 8082)), mock.call(manager, clis[1], ('127.0.0.3', 8083)), mock.call(manager, clis[2], ('127.0.0.4', 8084)), ]) acceptor.assert_has_calls([ mock.call(tendrils[0]), mock.call(tendrils[1]), mock.call(tendrils[2]), ]) self.assertFalse(clis[0].close.called) clis[1].close.assert_called_once_with() self.assertFalse(clis[2].close.called) mock_track_tendril.assert_has_calls([ mock.call(tendrils[0]), mock.call(tendrils[2]), ]) tendrils[0]._start.assert_called_once_with() self.assertFalse(tendrils[1]._start.called) tendrils[2]._start.assert_called_once_with() @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock(**{ 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril') def test_listener_acceptor_reject_connection(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): clis = [mock.Mock(), mock.Mock(), mock.Mock()] mock_socket.return_value.accept.side_effect = [ (clis[0], ('127.0.0.2', 8082)), (clis[1], ('127.0.0.3', 8083)), (clis[2], ('127.0.0.4', 8084)), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), ] tendrils = [mock.Mock(), mock.Mock(), mock.Mock()] mock_TCPTendril.side_effect = tendrils[:] acceptor = mock.Mock(side_effect=application.RejectConnection()) manager = tcp.TCPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) self.assertFalse(mock_sleep.called) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.listen(1024), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) mock_TCPTendril.assert_has_calls([ mock.call(manager, clis[0], ('127.0.0.2', 8082)), mock.call(manager, clis[1], ('127.0.0.3', 8083)), mock.call(manager, clis[2], ('127.0.0.4', 8084)), ]) acceptor.assert_has_calls([ mock.call(tendrils[0]), mock.call(tendrils[1]), mock.call(tendrils[2]), ]) clis[0].close.assert_called_once_with() clis[1].close.assert_called_once_with() clis[2].close.assert_called_once_with() self.assertFalse(mock_track_tendril.called) self.assertFalse(tendrils[0]._start.called) self.assertFalse(tendrils[1]._start.called) self.assertFalse(tendrils[2]._start.called) @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'getsockname.return_value': ('127.0.0.1', 8080), 'close.side_effect': socket.error(), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril') def test_listener_acceptor_badclose(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): clis = [ mock.Mock(**{ 'close.side_effect': socket.error(), }), mock.Mock(**{ 'close.side_effect': socket.error(), }), mock.Mock(**{ 'close.side_effect': socket.error(), }) ] mock_socket.return_value.accept.side_effect = [ (clis[0], ('127.0.0.2', 8082)), (clis[1], ('127.0.0.3', 8083)), (clis[2], ('127.0.0.4', 8084)), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), ] tendrils = [mock.Mock(), mock.Mock(), mock.Mock()] mock_TCPTendril.side_effect = tendrils[:] acceptor = mock.Mock(side_effect=[ mock.Mock(), TestException(), mock.Mock(), ]) manager = tcp.TCPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) self.assertFalse(mock_sleep.called) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.listen(1024), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) mock_TCPTendril.assert_has_calls([ mock.call(manager, clis[0], ('127.0.0.2', 8082)), mock.call(manager, clis[1], ('127.0.0.3', 8083)), mock.call(manager, clis[2], ('127.0.0.4', 8084)), ]) acceptor.assert_has_calls([ mock.call(tendrils[0]), mock.call(tendrils[1]), mock.call(tendrils[2]), ]) self.assertFalse(clis[0].close.called) clis[1].close.assert_called_once_with() self.assertFalse(clis[2].close.called) mock_track_tendril.assert_has_calls([ mock.call(tendrils[0]), mock.call(tendrils[2]), ]) tendrils[0]._start.assert_called_once_with() self.assertFalse(tendrils[1]._start.called) tendrils[2]._start.assert_called_once_with() @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock( **{ 'accept.side_effect': TestException(), 'getsockname.return_value': ('127.0.0.1', 8080), 'listen.side_effect': TestException(), 'close.side_effect': socket.error(), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock()) def test_listener_nolisten_noclose(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): acceptor = mock.Mock() manager = tcp.TCPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) self.assertFalse(mock_sleep.called) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.listen(1024), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) self.assertFalse(mock_TCPTendril.called) self.assertFalse(acceptor.called) self.assertFalse(mock_track_tendril.called) @mock.patch.object(gevent, 'sleep', side_effect=TestException()) @mock.patch.object(socket, 'socket', return_value=mock.Mock(**{ 'getsockname.return_value': ('127.0.0.1', 8080), })) @mock.patch.object(manager.TendrilManager, '_track_tendril') @mock.patch.object(tcp, 'TCPTendril') def test_listener_err_thresh(self, mock_TCPTendril, mock_track_tendril, mock_socket, mock_sleep): clis = [mock.Mock(), mock.Mock(), mock.Mock()] mock_socket.return_value.accept.side_effect = [ TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), TestException(), (clis[0], ('127.0.0.2', 8082)), (clis[1], ('127.0.0.3', 8083)), (clis[2], ('127.0.0.4', 8084)), TestException(), TestException(), TestException(), ] tendrils = [mock.Mock(), mock.Mock(), mock.Mock()] mock_TCPTendril.side_effect = tendrils[:] acceptor = mock.Mock(side_effect=[ mock.Mock(), TestException(), mock.Mock(), ]) manager = tcp.TCPTendrilManager() manager.running = True with self.assertRaises(TestException): manager.listener(acceptor, None) self.assertFalse(mock_sleep.called) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) mock_socket.return_value.assert_has_calls([ mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), mock.call.bind(('', 0)), mock.call.getsockname(), mock.call.listen(1024), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.accept(), mock.call.close(), ]) self.assertEqual(manager.local_addr, ('127.0.0.1', 8080)) mock_TCPTendril.assert_has_calls([ mock.call(manager, clis[0], ('127.0.0.2', 8082)), mock.call(manager, clis[1], ('127.0.0.3', 8083)), mock.call(manager, clis[2], ('127.0.0.4', 8084)), ]) acceptor.assert_has_calls([ mock.call(tendrils[0]), mock.call(tendrils[1]), mock.call(tendrils[2]), ]) self.assertFalse(clis[0].close.called) clis[1].close.assert_called_once_with() self.assertFalse(clis[2].close.called) mock_track_tendril.assert_has_calls([ mock.call(tendrils[0]), mock.call(tendrils[2]), ]) tendrils[0]._start.assert_called_once_with() self.assertFalse(tendrils[1]._start.called) tendrils[2]._start.assert_called_once_with()
def _exited_with(code): if code == -signal.SIGINT: return gevent.GreenletExit('Exited by SIGINT') return ProcessExit(code)