class ServerRack(object): def __init__(self, servers): self.servers = servers self.ev = Event() def start(self): started = [] try: for server in self.servers[:]: server.start() started.append(server) name = getattr(server, 'name', None) or server.__class__.__name__ or 'Server' except: self.stop(started) raise def serve_forever(self): self.start() self.ev.wait() def stop(self, servers=None): if servers is None: servers = self.servers[:] for server in servers: try: server.stop() except: if hasattr(server, 'loop'): #gevent >= 1.0 server.loop.handle_error(server.stop, *sys.exc_info()) else: # gevent <= 0.13 import traceback traceback.print_exc() self.ev.set()
class ConditionPoller(Thread): """ generic polling mechanism: every interval seconds, check if condition returns a true value. if so, pass the value to callback if condition or callback raise exception, stop polling. """ def __init__(self, condition, condition_callback, exception_callback, interval): self.polling_interval = interval self._shutdown_now = Event() self._condition = condition self._callback = condition_callback self._on_exception = exception_callback super(ConditionPoller,self).__init__() def shutdown(self): self.is_shutting_down = True self._shutdown_now.set() def run(self): try: while not self._shutdown_now.is_set(): self._check_condition() self._shutdown_now.wait(self.polling_interval) except: log.error('thread failed', exc_info=True) def _check_condition(self): try: value = self._condition() if value: self._callback(value) except Exception as e: log.debug('stopping poller after exception', exc_info=True) self.shutdown() if self._on_exception: self._on_exception(e) def start(self): super(ConditionPoller,self).start()
def events(): try: last = float(request.get_cookie('interconnect_last_event')) except Exception: last = time.time() evt = Event() events_history[0][1] > last and evt.set() event_waiters.add(evt) success = evt.wait(timeout=30) event_waiters.discard(evt) response.set_header('Content-Type', 'application/json') response.set_header('Pragma', 'no-cache') response.set_header('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate') response.set_header('Expires', 'Thu, 01 Dec 1994 16:00:00 GMT') success and response.set_cookie('interconnect_last_event', '%.5f' % time.time()) data = [] for e in events_history: if e[1] > last: data.append(e[0]) else: break data = list(reversed(data)) return json.dumps(data)
class BlackBerryPushService(object): def __init__(self, app_id, password, push_url): self.app_id = app_id self.password = password self.push_url = push_url self._send_queue = Queue() self._send_queue_cleared = Event() self.log = logging.getLogger('pulsus.service.bbp') def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: self.log.info("BlackBerry Push service started") while True: notification = self._send_queue.get() try: self._do_push(notification) except Exception, e: print e self._send_queue.put(notification) gevent.sleep(5.0) finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set()
class GServer(ProtoBufRPCServer): def __init__(self, host, port, service, poolsize=128): self.gpool = Pool(poolsize) self.stop_event = Event() context = zmq.Context() self.port = port self.socket = context.socket(zmq.ROUTER) self.socket.bind("tcp://%s:%s" % (host, port)) self.service = service def serve_forever(self,): while not self.stop_event.is_set(): try: msg = self.socket.recv_multipart() except zmq.ZMQError: if self.socket.closed: break raise e self.gpool.spawn(self.handle_request, msg) def shutdown(self,): self.socket.close() self.stop_event.set() def handle_request(self, msg): assert len(msg) == 3 (id_, null, request) = msg assert null == '' response = self.handle(request) self.socket.send_multipart([id_, null, response.SerializeToString()])
def run(ctx, dev): """Start the client""" # create app app = EthApp(ctx.obj['config']) if dev: gevent.get_hub().SYSTEM_ERROR = BaseException try: ctx.obj['config']['client_version'] += '/' + os.getlogin() except: log.warn("can't get and add login name to client_version") pass # register services for service in services: assert issubclass(service, BaseService) if service.name not in app.config['deactivated_services']: assert service.name not in app.services service.register_with_app(app) assert hasattr(app.services, service.name) # start app app.start() # wait for interupt evt = Event() gevent.signal(signal.SIGQUIT, evt.set) gevent.signal(signal.SIGTERM, evt.set) gevent.signal(signal.SIGINT, evt.set) evt.wait() # finally stop app.stop()
def test_greenlet(self): queue = JoinableQueue() requests_done = Event() g = Greenlet(self._producer, queue, FirstService(), 'Terminator') h = Greenlet(self._producer, queue, SecondService(), 'Terminator') i = Greenlet(self._producer, queue, ThirdService(), 'Terminator') requests = Group() for request in g, h, i: requests.add(request) log.debug('before spawn') c = spawn( self._consumer, done=requests_done, queue=queue, ) [r.start() for r in requests] log.debug('after spawn') requests.join() requests_done.set() log.debug('requests are done') c.join() log.debug('consumer is done')
def test_visual_transform(self): input_data_product_id = self.ctd_plain_input_data_product() output_data_product_id = self.google_dt_data_product() dpd = DataProcessDefinition(name='visual transform') dpd.data_process_type = DataProcessTypeEnum.TRANSFORM dpd.module = 'ion.processes.data.transforms.viz.google_dt' dpd.class_name = 'VizTransformGoogleDT' #-------------------------------------------------------------------------------- # Walk before we base jump #-------------------------------------------------------------------------------- data_process_definition_id = self.data_process_management.create_data_process_definition(dpd) self.addCleanup(self.data_process_management.delete_data_process_definition, data_process_definition_id) data_process_id = self.data_process_management.create_data_process2(data_process_definition_id=data_process_definition_id, in_data_product_ids=[input_data_product_id], out_data_product_ids=[output_data_product_id]) self.addCleanup(self.data_process_management.delete_data_process2,data_process_id) self.data_process_management.activate_data_process2(data_process_id) self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id) validated = Event() def validation(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) self.assertTrue(rdt['google_dt_components'] is not None) validated.set() self.setup_subscriber(output_data_product_id, callback=validation) self.publish_to_plain_data_product(input_data_product_id) self.assertTrue(validated.wait(10))
def __init__(self, config, etcd_api, status_reporter, hosts_ipset): super(_FelixEtcdWatcher, self).__init__(config.ETCD_ADDR, VERSION_DIR) self._config = config self._etcd_api = etcd_api self._status_reporter = status_reporter self.hosts_ipset = hosts_ipset # Keep track of the config loaded from etcd so we can spot if it # changes. self.last_global_config = None self.last_host_config = None self.my_config_dir = dir_for_per_host_config(self._config.HOSTNAME) # Events triggered by the EtcdAPI Actor to tell us to load the config # and start polling. These are one-way flags. self.load_config = Event() self.begin_polling = Event() # Event that we trigger once the config is loaded. self.configured = Event() # Polling state initialized at poll start time. self.splitter = None # Cache of known endpoints, used to resolve deletions of whole # directory trees. self.endpoint_ids_per_host = defaultdict(set) # Next-hop IP addresses of our hosts, if populated in etcd. self.ipv4_by_hostname = {} # Register for events when values change. self._register_paths()
def test_data_process_prime(self): self.lc_preload() instrument_data_product_id = self.ctd_instrument_data_product() derived_data_product_id = self.ctd_derived_data_product() data_process_id = self.data_process_management.create_data_process2(in_data_product_ids=[instrument_data_product_id], out_data_product_ids=[derived_data_product_id]) self.addCleanup(self.data_process_management.delete_data_process2, data_process_id) self.data_process_management.activate_data_process2(data_process_id) self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id) validated = Event() def validation(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) np.testing.assert_array_almost_equal(rdt['conductivity_L1'], np.array([42.914])) np.testing.assert_array_almost_equal(rdt['temp_L1'], np.array([20.])) np.testing.assert_array_almost_equal(rdt['pressure_L1'], np.array([3.068])) np.testing.assert_array_almost_equal(rdt['density'], np.array([1021.7144739593881])) np.testing.assert_array_almost_equal(rdt['salinity'], np.array([30.935132729668283])) validated.set() self.setup_subscriber(derived_data_product_id, callback=validation) self.publish_to_data_product(instrument_data_product_id) self.assertTrue(validated.wait(10))
def test_actors(self): input_data_product_id = self.ctd_plain_input_data_product() output_data_product_id = self.ctd_plain_density() actor = self.create_density_transform_function() route = {input_data_product_id: {output_data_product_id: actor}} config = DotDict() config.process.routes = route config.process.params.lat = 45. config.process.params.lon = -71. data_process_id = self.data_process_management.create_data_process2(in_data_product_ids=[input_data_product_id], out_data_product_ids=[output_data_product_id], configuration=config) self.addCleanup(self.data_process_management.delete_data_process2, data_process_id) self.data_process_management.activate_data_process2(data_process_id) self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id) validated = Event() def validation(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) # The value I use is a double, the value coming back is only a float32 so there's some data loss but it should be precise to the 4th digit np.testing.assert_array_almost_equal(rdt['density'], np.array([1021.6839775385847]), decimal=4) validated.set() self.setup_subscriber(output_data_product_id, callback=validation) self.publish_to_plain_data_product(input_data_product_id) self.assertTrue(validated.wait(10))
def test_with_remoting(defer): random_data = str(random.randint(0, 10000000000)) node1 = Node('localhost:20001', enable_remoting=True) defer(node1.stop) node2 = Node('localhost:20002', enable_remoting=True) defer(node2.stop) f_src = tempfile.NamedTemporaryFile() f_src.write(random_data) f_src.flush() class Sender(Actor): def run(self, receiver): ref = serve_file(f_src.name) receiver << ref class Receiver(Actor): def receive(self, fref): for _ in range(2): fetched_path = fref.fetch() with open(fetched_path) as f_dst: eq_(random_data, f_dst.read()) received.set() received = Event() node2.spawn(Receiver, name='receiver') receiver_ref = node1.lookup_str('localhost:20002/receiver') node1.spawn(Sender.using(receiver=receiver_ref)) received.wait()
def test_reentrant(self): pool = self.klass(1) result = pool.apply(pool.apply, (lambda a: a + 1, (5, ))) self.assertEqual(result, 6) evt = Event() pool.apply_async(evt.set) evt.wait()
class Worker(object): ''' 子进程运行的代码,通过起一个协程来和主进程通信 包括接受任务分配请求,退出信号(零字节包),及反馈任务执行进度 然后主协程等待停止信号并中止进程(stop_event用于协程间同步)。 ''' def __init__(self, url): self.url = url self.stop_event = Event() gevent.spawn(self.communicate) self.stop_event.wait() print 'worker(%s):will stop' % os.getpid() def exec_task(self, task): print 'worker(%s):execute task:%s' % (os.getpid(), task.rstrip('\n')) def communicate(self): print 'worker(%s):started' % os.getpid() client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client.connect(self.url) fp = client.makefile() while True: line = fp.readline() if not line: self.stop_event.set() break '单独起一个协程去执行任务,防止通信协程阻塞' gevent.spawn(self.exec_task, line)
def test_heartbeat_with_listeners(self): mocklistener = Mock(spec=ProcessRPCServer) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc) readyev = Event() readyev.set() mocklistener.get_ready_event.return_value = readyev def fake_listen(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() mocklistener.listen = lambda *a, **kw: fake_listen(listenoutev, listeninev) p.start() p.get_ready_event().wait(timeout=5) p.start_listeners() listenoutev.wait(timeout=5) # wait for listen loop to start self.addCleanup(listeninev.set) # makes listen loop fall out on shutdown self.addCleanup(p.stop) # now test heartbeat! hb = p.heartbeat() self.assertEquals((True, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)
def test_qc_events(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_qc_pdict() stream_def_id = self.pubsub_management.create_stream_definition('qc stream def', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('qc stream', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) ingestion_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) config = DotDict() self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id, config=config) self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id) publisher = StandaloneStreamPublisher(stream_id, route) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) rdt['temp'] = np.arange(10) * 3 verified = Event() def verification(event, *args, **kwargs): self.assertEquals(event.qc_parameter, 'temp_qc') self.assertEquals(event.temporal_value, 7) verified.set() es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=dataset_id, callback=verification, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(verified.wait(10))
def test_ingestion_failover(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) event = Event() def cb(*args, **kwargs): event.set() sub = EventSubscriber(event_type="ExceptionEvent", callback=cb, origin="stream_exception") sub.start() self.publish_fake_data(stream_id, route) self.wait_until_we_have_enough_granules(dataset_id, 40) file_path = DatasetManagementService._get_coverage_path(dataset_id) master_file = os.path.join(file_path, '%s_master.hdf5' % dataset_id) with open(master_file, 'w') as f: f.write('this will crash HDF') self.publish_hifi(stream_id, route, 5) self.assertTrue(event.wait(10)) sub.stop()
def test_execute_advanced_transform(self): # Runs a transform across L0-L2 with stream definitions including available fields streams = self.setup_advanced_transform() in_stream_id, in_stream_def_id = streams[0] out_stream_id, out_stream_defs_id = streams[1] validation_event = Event() def validator(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) if not np.allclose(rdt['rho'], np.array([1001.0055034])): return validation_event.set() self.setup_validator(validator) in_route = self.pubsub_management.read_stream_route(in_stream_id) publisher = StandaloneStreamPublisher(in_stream_id, in_route) outbound_rdt = RecordDictionaryTool(stream_definition_id=in_stream_def_id) outbound_rdt['time'] = [0] outbound_rdt['TEMPWAT_L0'] = [280000] outbound_rdt['CONDWAT_L0'] = [100000] outbound_rdt['PRESWAT_L0'] = [2789] outbound_rdt['lat'] = [45] outbound_rdt['lon'] = [-71] outbound_granule = outbound_rdt.to_granule() publisher.publish(outbound_granule) self.assertTrue(validation_event.wait(2))
class Console(BaseService): """A service starting an interactive ipython session when receiving the SIGSTP signal (e.g. via keyboard shortcut CTRL-Z). """ name = 'console' def __init__(self, app): super(Console, self).__init__(app) self.interrupt = Event() gevent.signal(signal.SIGTSTP, self.interrupt.set) self.console_locals = [] def start(self): super(Console, self).start() self.console_locals = {} self.console_locals.update(self.app.services) self.console_locals['app'] = self.app def _run(self): while True: self.interrupt.wait() IPython.start_ipython(argv=['--gui', 'gevent'], user_ns=self.console_locals) self.interrupt.clear()
class InputStream(object): """ FCGI_STDIN or FCGI_DATA stream. Uses temporary file to store received data once max_mem bytes have been received. """ def __init__(self, max_mem=1024): self._file = SpooledTemporaryFile(max_mem) self._eof_received = Event() def feed(self, data): if self._eof_received.is_set(): raise IOError('Feeding file beyond EOF mark') if not data: # EOF mark self._file.seek(0) self._eof_received.set() else: self._file.write(data) def __iter__(self): self._eof_received.wait() return iter(self._file) def read(self, size=-1): self._eof_received.wait() return self._file.read(size) def readlines(self, sizehint=0): self._eof_received.wait() return self._file.readlines(sizehint) @property def eof_received(self): return self._eof_received.is_set()
def __init__(self, sessid, manager, config, error_handler=None): self.manager = weakref.proxy(manager) self.sessid = sessid self.session = manager.make_session(sessid) # the session dict, for general developer usage for qname in QUEUE_NAMES: setattr(self, qname, manager.make_queue(sessid, qname)) self.hits = 0 self.heartbeats = 0 self.hb_check_timeout = Event() self.hb_send_timeout = Event() self.wsgi_app_greenlet = None self.state = "NEW" self.connection_established = False self.ack_callbacks = {} self.ack_counter = 0 self.request = None self.environ = None self.namespaces = {} self.active_ns = {} # Namespace sessions that were instantiated self.jobs = [] self.error_handler = default_error_handler self.config = config if error_handler is not None: self.error_handler = error_handler
def test_data_product_subscription(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) tdom, sdom = time_series_domain() dp = DataProduct(name='ctd parsed') dp.spatial_domain = sdom.dump() dp.temporal_domain = tdom.dump() data_product_id = self.data_product_management.create_data_product(data_product=dp, stream_definition_id=stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) subscription_id = self.pubsub_management.create_subscription('validator', data_product_ids=[data_product_id]) self.addCleanup(self.pubsub_management.delete_subscription, subscription_id) validated = Event() def validation(msg, route, stream_id): validated.set() stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True) dp_stream_id = stream_ids.pop() validator = StandaloneStreamSubscriber('validator', callback=validation) validator.start() self.addCleanup(validator.stop) self.pubsub_management.activate_subscription(subscription_id) self.addCleanup(self.pubsub_management.deactivate_subscription, subscription_id) route = self.pubsub_management.read_stream_route(dp_stream_id) publisher = StandaloneStreamPublisher(dp_stream_id, route) publisher.publish('hi') self.assertTrue(validated.wait(10))
def test_serialize_compatability(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('ctd extended', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('ctd1', 'xp1', stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) sub_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id]) self.addCleanup(self.pubsub_management.delete_subscription, sub_id) self.pubsub_management.activate_subscription(sub_id) self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id) verified = Event() def verifier(msg, route, stream_id): for k,v in msg.record_dictionary.iteritems(): if v is not None: self.assertIsInstance(v, np.ndarray) rdt = RecordDictionaryTool.load_from_granule(msg) for k,v in rdt.iteritems(): self.assertIsInstance(rdt[k], np.ndarray) self.assertIsInstance(v, np.ndarray) verified.set() subscriber = StandaloneStreamSubscriber('sub1', callback=verifier) subscriber.start() self.addCleanup(subscriber.stop) publisher = StandaloneStreamPublisher(stream_id,route) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) ph.fill_rdt(rdt,10) publisher.publish(rdt.to_granule()) self.assertTrue(verified.wait(60))
def run(ctx, dev, nodial, fake): """Start the client ( --dev to stop on error)""" config = ctx.obj['config'] if nodial: # config['deactivated_services'].append(PeerManager.name) # config['deactivated_services'].append(NodeDiscovery.name) config['discovery']['bootstrap_nodes'] = [] config['discovery']['listen_port'] = 29873 config['p2p']['listen_port'] = 29873 config['p2p']['min_peers'] = 0 if fake: from ethereum import blocks blocks.GENESIS_DIFFICULTY = 1024 blocks.BLOCK_DIFF_FACTOR = 16 blocks.MIN_GAS_LIMIT = blocks.GENESIS_GAS_LIMIT / 2 # workaround for genesis.json hack blocks.GENESIS_JSON["difficulty"] = blocks.int_to_hex(blocks.GENESIS_DIFFICULTY) # create app app = EthApp(config) # development mode if dev: gevent.get_hub().SYSTEM_ERROR = BaseException try: config['client_version'] += '/' + os.getlogin() except: log.warn("can't get and add login name to client_version") pass # dump config konfig.dump_config(config) # register services for service in services: assert issubclass(service, BaseService) if service.name not in app.config['deactivated_services']: assert service.name not in app.services service.register_with_app(app) assert hasattr(app.services, service.name) unlock_accounts(ctx.obj['unlock'], app.services.accounts, password=ctx.obj['password']) # start app log.info('starting') app.start() if config['post_app_start_callback'] is not None: config['post_app_start_callback'](app) # wait for interrupt evt = Event() gevent.signal(signal.SIGQUIT, evt.set) gevent.signal(signal.SIGTERM, evt.set) gevent.signal(signal.SIGINT, evt.set) evt.wait() # finally stop app.stop()
def handle(): connection = create_postgresql_connection() cursor = connection.cursor() cursor.execute("BEGIN;") cursor.execute("DELETE FROM core_ratequery;") cursor.execute("COMMIT;") cursor.close() queue = JoinableQueue() event = Event() age_ids = age_map(connection).values() + [None] sex_ids = sex_map(connection).values() + [None] education_ids = education_map(connection).values() + [None] province_ids = province_map(connection).values() + [None] cursor = connection.cursor() cursor.execute("SELECT DISTINCT cycle FROM core_microdata;"); cycles = [row[0] for row in cursor] cursor.close() greenlets = [] for i in range(50): gv = gevent.spawn(worker, queue, event) greenlets.append(gv) combs = itertools.product(age_ids, sex_ids, province_ids, education_ids, cycles) for c in combs: queue.put(c) queue.join() event.set() gevent.joinall(greenlets)
def __init__(self): # # Dictionary: key is a websocket, value is a name # self.ws_dict = {} # # Dictionary: key is a name, value is a websocket # self.ws_dict_chan = {} # # Dictionary: key is a channel name, value is a list of groupDevices # self.channels_to_groupDevices = {} self.uber_client_ws_client = "" self.uber_client_ws_server = "" self.WS_PRIVATE_PORT = 8111 self.WS_PORT = 8112 self.uber_client_ready = Event() self.private_server_ready = Event() self.uber_server_socket_ready = Event() self.public_ready = Event() self.uber_client_subscriptions_queue = Queue() self.DEBUG_INTERNAL = DEBUG_INTERNAL self.DEBUG_EXTERNAL = DEBUG_EXTERNAL self.DEBUG_OVERRIDE = DEBUG_OVERRIDE self._lock = gevent.lock.RLock() print("Initialized WearConnectServer")
def evt_user_input(cls, arg): trans, ilet = arg evt = Event() ilet.event = evt process_msg(('evt_user_input', arg)) evt.wait() return ilet
class Pinger(Greenlet): """ Very simple test 'app' """ def __init__(self, id): super(Pinger,self).__init__() self.event = Event() self.conn = None self.id = id def _run(self): logger.debug("Pinger starting") self.conn = connection.AMQPConnection(self) self.conn.connect() #self.conn.connection.join() self.event.wait() logger.debug("Pinger exiting") self.amqp.close() self.conn.close() def on_connect(self, connection): self.amqp = PingerAMQPManager(connection, self, self.id) def handle_message(self, message): if message.routing_key.endswith('pinger.exit'): #self.conn.connection.close() self.event.set()
class C2DMService(object): def __init__(self, source, email, password): self.source = source self.email = email self.password = password self._send_queue = Queue() self._send_queue_cleared = Event() self.log = logging.getLogger('pulsus.service.c2dm') def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: self.log.info("C2DM service started") while True: notification = self._send_queue.get() try: self._do_push(notification) except Exception, e: self.log.exception("Error while pushing") self._send_queue.put(notification) gevent.sleep(5.0) finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set()
class ClusterCoordinator(Service): port = Setting('cluster_port', default=4440) def __init__(self, identity, leader=None, cluster=None): leader = leader or identity self.server = PeerServer(self, identity) self.client = PeerClient(self, leader, identity) self.set = cluster or ObservableSet() self.promoted = Event() self.add_service(self.server) if leader != identity: self.add_service(self.client) self.is_leader = False else: self.is_leader = True def wait_for_promotion(self): self.promoted.wait() @property def leader(self): return self.client.leader @property def identity(self): return self.client.identity
class uWSGIWebSocket(object): # pragma: no cover """ This wrapper class provides a uWSGI WebSocket interface that is compatible with eventlet's implementation. """ def __init__(self, app): self.app = app def __call__(self, environ, start_response): self.environ = environ uwsgi.websocket_handshake() self._req_ctx = None if hasattr(uwsgi, 'request_context'): # uWSGI >= 2.1.x with support for api access across-greenlets self._req_ctx = uwsgi.request_context() else: # use event and queue for sending messages from gevent.event import Event from gevent.queue import Queue from gevent.select import select self._event = Event() self._send_queue = Queue() # spawn a select greenlet def select_greenlet_runner(fd, event): """Sets event when data becomes available to read on fd.""" while True: event.set() try: select([fd], [], [])[0] except ValueError: break self._select_greenlet = gevent.spawn(select_greenlet_runner, uwsgi.connection_fd(), self._event) self.app(self) def close(self): """Disconnects uWSGI from the client.""" uwsgi.disconnect() if self._req_ctx is None: # better kill it here in case wait() is not called again self._select_greenlet.kill() self._event.set() def _send(self, msg): """Transmits message either in binary or UTF-8 text mode, depending on its type.""" if isinstance(msg, six.binary_type): method = uwsgi.websocket_send_binary else: method = uwsgi.websocket_send if self._req_ctx is not None: method(msg, request_context=self._req_ctx) else: method(msg) def _decode_received(self, msg): """Returns either bytes or str, depending on message type.""" if not isinstance(msg, six.binary_type): # already decoded - do nothing return msg # only decode from utf-8 if message is not binary data type = six.byte2int(msg[0:1]) if type >= 48: # no binary return msg.decode('utf-8') # binary message, don't try to decode return msg def send(self, msg): """Queues a message for sending. Real transmission is done in wait method. Sends directly if uWSGI version is new enough.""" if self._req_ctx is not None: self._send(msg) else: self._send_queue.put(msg) self._event.set() def wait(self): """Waits and returns received messages. If running in compatibility mode for older uWSGI versions, it also sends messages that have been queued by send(). A return value of None means that connection was closed. This must be called repeatedly. For uWSGI < 2.1.x it must be called from the main greenlet.""" while True: if self._req_ctx is not None: try: msg = uwsgi.websocket_recv(request_context=self._req_ctx) except IOError: # connection closed return None return self._decode_received(msg) else: # we wake up at least every 3 seconds to let uWSGI # do its ping/ponging event_set = self._event.wait(timeout=3) if event_set: self._event.clear() # maybe there is something to send msgs = [] while True: try: msgs.append(self._send_queue.get(block=False)) except gevent.queue.Empty: break for msg in msgs: self._send(msg) # maybe there is something to receive, if not, at least # ensure uWSGI does its ping/ponging try: msg = uwsgi.websocket_recv_nb() except IOError: # connection closed self._select_greenlet.kill() return None if msg: # message available return self._decode_received(msg)
def __init__(self): self.read = [] self.write = [] self.event = Event()
def __init__(self): self.events = set() self.event = Event()
#coding=utf8 import gevent from gevent.event import Event ''' 事件 ''' evt = Event() def setter(): '''After 3 seconds, wake all threads waiting on the value of evt''' print('A: Hey wait for me, I have to do something') gevent.sleep(3) print("Ok, I'm done") evt.set() def waiter(): '''After 3 seconds the get call will unblock''' print("I'll wait for you") evt.wait() # blocking print("It's about time") def main(): gevent.joinall([ gevent.spawn(setter), gevent.spawn(waiter), gevent.spawn(waiter), gevent.spawn(waiter),
class MonitorNetwork(Jobmanager, NodeMonitorMixin): one_min_stats = ['work_restarts', 'new_jobs', 'work_pushes'] defaults = config = dict(coinservs=REQUIRED, diff1=0x0000FFFF00000000000000000000000000000000000000000000000000000000, hashes_per_share=0xFFFF, merged=tuple(), block_poll=0.2, job_refresh=15, rpc_ping_int=2, pow_block_hash=False, poll=None, currency=REQUIRED, algo=REQUIRED, pool_address='', signal=None, payout_drk_mn=True, max_blockheight=None) def __init__(self, config): NodeMonitorMixin.__init__(self) self._configure(config) if get_bcaddress_version(self.config['pool_address']) is None: raise ConfigurationError("No valid pool address configured! Exiting.") # Since some MonitorNetwork objs are polling and some aren't.... self.gl_methods = ['_monitor_nodes', '_check_new_jobs'] # Aux network monitors (merged mining) self.auxmons = [] # internal vars self._last_gbt = {} self._job_counter = 0 # a unique job ID counter # Currently active jobs keyed by their unique ID self.jobs = {} self.latest_job = None # The last job that was generated self.new_job = Event() self.last_signal = 0.0 # general current network stats self.current_net = dict(difficulty=None, height=None, last_block=0.0, prev_hash=None, transactions=None, subsidy=None) self.block_stats = dict(accepts=0, rejects=0, solves=0, last_solve_height=None, last_solve_time=None, last_solve_worker=None) self.recent_blocks = deque(maxlen=15) # Run the looping height poller if we aren't getting push notifications if (not self.config['signal'] and self.config['poll'] is None) or self.config['poll']: self.gl_methods.append('_poll_height') @property def status(self): """ For display in the http monitor """ ret = dict(net_state=self.current_net, block_stats=self.block_stats, last_signal=self.last_signal, currency=self.config['currency'], live_coinservers=len(self._live_connections), down_coinservers=len(self._down_connections), coinservers={}, job_count=len(self.jobs)) for connection in self._live_connections: st = connection.status() st['status'] = 'live' ret['coinservers'][connection.name] = st for connection in self._down_connections: st = connection.status() st['status'] = 'down' ret['coinservers'][connection.name] = st return ret def start(self): Jobmanager.start(self) if self.config['signal']: self.logger.info("Listening for push block notifs on signal {}" .format(self.config['signal'])) gevent.signal(self.config['signal'], self.getblocktemplate, signal=True) # Find desired auxmonitors self.config['merged'] = set(self.config['merged']) found_merged = set() for mon in self.manager.component_types['Jobmanager']: if mon.key in self.config['merged']: self.auxmons.append(mon) found_merged.add(mon.key) mon.new_job.rawlink(self.new_merged_work) for monitor in self.config['merged'] - found_merged: self.logger.error("Unable to locate Auxmonitor(s) '{}'".format(monitor)) def found_block(self, raw_coinbase, address, worker, hash_hex, header, job, start): """ Submit a valid block (hopefully!) to the RPC servers """ block = hexlify(job.submit_serial(header, raw_coinbase=raw_coinbase)) result = {} def record_outcome(success): # If we've already recorded a result, then return if result: return if start: submission_time = time.time() - start self.logger.info( "Recording block submission outcome {} after {}" .format(success, submission_time)) if success: self.manager.log_event( "{name}.block_submission_{curr}:{t}|ms" .format(name=self.manager.config['procname'], curr=self.config['currency'], t=submission_time * 1000)) if success: self.block_stats['accepts'] += 1 self.recent_blocks.append( dict(height=job.block_height, timestamp=int(time.time()))) else: self.block_stats['rejects'] += 1 self.logger.info("{} BLOCK {}:{} REJECTED" .format(self.config['currency'], hash_hex, job.block_height)) result.update(dict( address=address, height=job.block_height, total_subsidy=job.total_value, fees=job.fee_total, hex_bits=hexlify(job.bits), hex_hash=hash_hex, worker=worker, algo=job.algo, merged=False, success=success, currency=self.config['currency'] )) def submit_block(conn): retries = 0 while retries < 5: retries += 1 res = "failed" try: res = conn.submitblock(block) except (CoinRPCException, socket.error, ValueError) as e: self.logger.info("Block failed to submit to the server {} with submitblock! {}" .format(conn.name, e)) if getattr(e, 'error', {}).get('code', 0) != -8: self.logger.error(getattr(e, 'error'), exc_info=True) try: res = conn.getblocktemplate({'mode': 'submit', 'data': block}) except (CoinRPCException, socket.error, ValueError) as e: self.logger.error("Block failed to submit to the server {}!" .format(conn.name), exc_info=True) self.logger.error(getattr(e, 'error')) if res is None: self.logger.info("{} BLOCK {}:{} accepted by {}" .format(self.config['currency'], hash_hex, job.block_height, conn.name)) record_outcome(True) break # break retry loop if success else: self.logger.error( "Block failed to submit to the server {}, " "server returned {}!".format(conn.name, res), exc_info=True) sleep(1) self.logger.info("Retry {} for connection {}".format(retries, conn.name)) for tries in xrange(200): if not self._live_connections: self.logger.error("No live connections to submit new block to!" " Retry {} / 200.".format(tries)) sleep(0.1) continue gl = [] for conn in self._live_connections: # spawn a new greenlet for each submission to do them all async. # lower orphan chance gl.append(spawn(submit_block, conn)) gevent.joinall(gl) # If none of the submission threads were successfull then record a # failure if not result: record_outcome(False) break self.logger.log(35, "Valid network block identified!") self.logger.info("New block at height {} with hash {} and subsidy {}" .format(job.block_height, hash_hex, job.total_value)) self.block_stats['solves'] += 1 self.block_stats['last_solve_hash'] = hash_hex self.block_stats['last_solve_height'] = job.block_height self.block_stats['last_solve_worker'] = "{}.{}".format(address, worker) self.block_stats['last_solve_time'] = datetime.datetime.utcnow() if __debug__: self.logger.debug("New block hex dump:\n{}".format(block)) self.logger.debug("Coinbase: {}".format(str(job.coinbase.to_dict()))) for trans in job.transactions: self.logger.debug(str(trans.to_dict())) # Pass back all the results to the reporter who's waiting return result @loop(interval='block_poll') def _poll_height(self): try: height = self.call_rpc('getblockcount') except RPCException: return if self.current_net['height'] != height: self.logger.info("New block on main network detected with polling") self.current_net['height'] = height self.getblocktemplate(new_block=True) @loop(interval='job_refresh') def _check_new_jobs(self): self.getblocktemplate() def getblocktemplate(self, new_block=False, signal=False): if signal: self.last_signal = time.time() try: # request local memory pool and load it in bt = self.call_rpc('getblocktemplate', {'capabilities': [ 'coinbasevalue', 'coinbase/append', 'coinbase', 'generation', 'time', 'transactions/remove', 'prevblock', ]}) except RPCException: return False if self._last_gbt.get('height') != bt['height']: new_block = True # If this was from a push signal and the if signal and new_block: self.logger.info("Push block signal notified us of a new block!") elif signal: self.logger.info("Push block signal notified us of a block we " "already know about!") return # generate a new job if we got some new work! dirty = False if bt != self._last_gbt: self._last_gbt = bt self._last_gbt['update_time'] = time.time() dirty = True if new_block or dirty: # generate a new job and push it if there's a new block on the # network self.generate_job(push=new_block, flush=new_block, new_block=new_block) def new_merged_work(self, event): self.generate_job(push=True, flush=event.flush, network='aux') def generate_job(self, push=False, flush=False, new_block=False, network='main'): """ Creates a new job for miners to work on. Push will trigger an event that sends new work but doesn't force a restart. If flush is true a job restart will be triggered. """ # aux monitors will often call this early when not needed at startup if not self._last_gbt: self.logger.warn("Cannot generate new job, missing last GBT info") return if self.auxmons: merged_work = {} auxdata = {} for auxmon in self.auxmons: if auxmon.last_work['hash'] is None: continue merged_work[auxmon.last_work['chainid']] = dict( hash=auxmon.last_work['hash'], target=auxmon.last_work['type'] ) tree, size = bitcoin_data.make_auxpow_tree(merged_work) mm_hashes = [merged_work.get(tree.get(i), dict(hash=0))['hash'] for i in xrange(size)] mm_data = '\xfa\xbemm' mm_data += bitcoin_data.aux_pow_coinbase_type.pack(dict( merkle_root=bitcoin_data.merkle_hash(mm_hashes), size=size, nonce=0, )) for auxmon in self.auxmons: if auxmon.last_work['hash'] is None: continue data = dict(target=auxmon.last_work['target'], hash=auxmon.last_work['hash'], height=auxmon.last_work['height'], found_block=auxmon.found_block, index=mm_hashes.index(auxmon.last_work['hash']), type=auxmon.last_work['type'], hashes=mm_hashes) auxdata[auxmon.config['currency']] = data else: auxdata = {} mm_data = None # here we recalculate the current merkle branch and partial # coinbases for passing to the mining clients coinbase = Transaction() coinbase.version = 2 # create a coinbase input with encoded height and padding for the # extranonces so script length is accurate extranonce_length = (self.manager.config['extranonce_size'] + self.manager.config['extranonce_serv_size']) coinbase.inputs.append( Input.coinbase(self._last_gbt['height'], addtl_push=[mm_data] if mm_data else [], extra_script_sig=b'\0' * extranonce_length)) # Payout Darkcoin masternodes mn_enforcement = self._last_gbt.get('enforce_masternode_payments', True) if (self.config['payout_drk_mn'] is True or mn_enforcement is True) \ and self._last_gbt.get('payee', '') != '': # Grab the darkcoin payout amount, default to 20% payout = self._last_gbt.get('payee_amount', self._last_gbt['coinbasevalue'] / 5) self._last_gbt['coinbasevalue'] -= payout coinbase.outputs.append( Output.to_address(payout, self._last_gbt['payee'])) self.logger.info("Paying out masternode at addr {}. Payout {}. Blockval reduced to {}" .format(self._last_gbt['payee'], payout, self._last_gbt['coinbasevalue'])) # simple output to the proper address and value coinbase.outputs.append( Output.to_address(self._last_gbt['coinbasevalue'], self.config['pool_address'])) job_id = hexlify(struct.pack(str("I"), self._job_counter)) bt_obj = BlockTemplate.from_gbt(self._last_gbt, coinbase, extranonce_length, [Transaction(unhexlify(t['data']), fees=t['fee']) for t in self._last_gbt['transactions']]) # add in our merged mining data if mm_data: hashes = [bitcoin_data.hash256(tx.raw) for tx in bt_obj.transactions] bt_obj.merkle_link = bitcoin_data.calculate_merkle_link([None] + hashes, 0) bt_obj.merged_data = auxdata bt_obj.job_id = job_id bt_obj.diff1 = self.config['diff1'] bt_obj.algo = self.config['algo'] bt_obj.currency = self.config['currency'] bt_obj.pow_block_hash = self.config['pow_block_hash'] bt_obj.block_height = self._last_gbt['height'] bt_obj.acc_shares = set() bt_obj.flush = flush bt_obj.found_block = self.found_block # Push the fresh job to users after updating details self._job_counter += 1 if flush: self.jobs.clear() self.jobs[job_id] = bt_obj self.latest_job = bt_obj if push or flush: self.new_job.job = bt_obj self.new_job.set() self.new_job.clear() self.logger.info("{}: New block template with {:,} trans. " "Diff {:,.4f}. Subsidy {:,.2f}. Height {:,}. " "Merged: {}" .format("FLUSH" if flush else "PUSH", len(self._last_gbt['transactions']), bits_to_difficulty(self._last_gbt['bits']), self._last_gbt['coinbasevalue'] / 100000000.0, self._last_gbt['height'], ', '.join(auxdata.keys()))) # Stats and notifications now that it's pushed if flush: self._incr('work_restarts') self._incr('work_pushes') self.logger.info("New {} network block announced! Wiping previous" " jobs and pushing".format(network)) elif push: self.logger.info("New {} network block announced, pushing new job!" .format(network)) self._incr('work_pushes') if new_block: hex_bits = hexlify(bt_obj.bits) self.current_net['difficulty'] = bits_to_difficulty(hex_bits) self.current_net['subsidy'] = bt_obj.total_value self.current_net['height'] = bt_obj.block_height - 1 self.current_net['last_block'] = time.time() self.current_net['prev_hash'] = bt_obj.hashprev_be_hex self.current_net['transactions'] = len(bt_obj.transactions) self.manager.log_event( "{name}.{curr}.difficulty:{diff}|g\n" "{name}.{curr}.subsidy:{subsidy}|g\n" "{name}.{curr}.job_generate:{t}|g\n" "{name}.{curr}.height:{height}|g" .format(name=self.manager.config['procname'], curr=self.config['currency'], diff=self.current_net['difficulty'], subsidy=bt_obj.total_value, height=bt_obj.block_height - 1, t=(time.time() - self._last_gbt['update_time']) * 1000)) self._incr('new_jobs')
class PriorityQueue(object): """A priority queue. It is greenlet-safe, and offers the ability of changing priorities and removing arbitrary items. The queue is implemented as a custom min-heap. The priority is a mix of a discrete priority level and of the timestamp. The elements of the queue are QueueItems. """ PRIORITY_EXTRA_HIGH = 0 PRIORITY_HIGH = 1 PRIORITY_MEDIUM = 2 PRIORITY_LOW = 3 PRIORITY_EXTRA_LOW = 4 def __init__(self): """Create a priority queue.""" # The queue: a min-heap whose elements are of the form # (priority, timestamp, item), where item is the actual data. self._queue = [] # Reverse lookup for the items in the queue: a dictionary # associating the index in the queue to each item. self._reverse = {} # Event to signal that there are items in the queue. self._event = Event() # Index of the next element that will be added to the queue. self._next_index = 0 def __len__(self): return len(self._queue) def _verify(self): """Make sure that the internal state of the queue is consistent. This is used only for testing. """ if len(self._queue) != len(self._reverse): return False if len(self._queue) != self.length(): return False if self.empty() != (self.length() == 0): return False if self._event.isSet() == self.empty(): return False for item, idx in iteritems(self._reverse): if self._queue[idx].item != item: return False return True def __contains__(self, item): """Implement the 'in' operator for an item in the queue. item (QueueItem): an item to search. return (bool): True if item is in the queue. """ return item in self._reverse def _swap(self, idx1, idx2): """Swap two elements in the queue, keeping their reverse indices up to date. idx1 (int): the index of the first element. idx2 (int): the index of the second element. """ self._queue[idx1], self._queue[idx2] = \ self._queue[idx2], self._queue[idx1] self._reverse[self._queue[idx1].item] = idx1 self._reverse[self._queue[idx2].item] = idx2 def _up_heap(self, idx): """Take the element in position idx up in the heap until its position is the right one. idx (int): the index of the element to lift. return (int): the new index of the element. """ while idx > 0: parent = (idx - 1) // 2 if self._queue[idx] < self._queue[parent]: self._swap(parent, idx) idx = parent else: break return idx def _down_heap(self, idx): """Take the element in position idx down in the heap until its position is the right one. idx (int): the index of the element to lower. return (int): the new index of the element. """ last = len(self._queue) - 1 while 2 * idx + 1 <= last: child = 2 * idx + 1 if 2 * idx + 2 <= last and \ self._queue[2 * idx + 2] < self._queue[child]: child = 2 * idx + 2 if self._queue[child] < self._queue[idx]: self._swap(child, idx) idx = child else: break return idx def _updown_heap(self, idx): """Perform both operations of up_heap and down_heap on an element. idx (int): the index of the element to lift. return (int): the new index of the element. """ idx = self._up_heap(idx) return self._down_heap(idx) def push(self, item, priority=None, timestamp=None): """Push an item in the queue. If timestamp is not specified, uses the current time. item (QueueItem): the item to add to the queue. priority (int|None): the priority of the item, or None for medium priority. timestamp (datetime|None): the time of the submission, or None to use now. return (bool): false if the element was already in the queue and was not pushed again, true otherwise.. """ if item in self._reverse: return False if priority is None: priority = PriorityQueue.PRIORITY_MEDIUM if timestamp is None: timestamp = make_datetime() index = self._next_index self._next_index += 1 self._queue.append(QueueEntry(item, priority, timestamp, index)) last = len(self._queue) - 1 self._reverse[item] = last self._up_heap(last) # Signal to listener greenlets that there might be something. self._event.set() return True def top(self, wait=False): """Return the first element in the queue without extracting it. wait (bool): if True, block until an element is present. return (QueueEntry): first element in the queue. raise (LookupError): on empty queue if wait was false. """ if not self.empty(): return self._queue[0] else: if not wait: raise LookupError("Empty queue.") else: while True: if self.empty(): self._event.wait() continue return self._queue[0] def pop(self, wait=False): """Extract (and return) the first element in the queue. wait (bool): if True, block until an element is present. return (QueueEntry): first element in the queue. raise (LookupError): on empty queue, if wait was false. """ top = self.top(wait) last = len(self._queue) - 1 self._swap(0, last) del self._reverse[top.item] del self._queue[last] # last is 0 when the queue becomes empty. if last > 0: self._down_heap(0) else: # Signal that there is nothing left for listeners. self._event.clear() return top def remove(self, item): """Remove an item from the queue. Raise a KeyError if not present. item (QueueItem): the item to remove. return (QueueEntry): the complete entry removed. raise (KeyError): if item not present. """ pos = self._reverse[item] entry = self._queue[pos] last = len(self._queue) - 1 self._swap(pos, last) del self._reverse[item] del self._queue[last] if pos != last: self._updown_heap(pos) if self.empty(): self._event.clear() return entry def set_priority(self, item, priority): """Change the priority of an item inside the queue. Raises an exception if the item is not in the queue. item (QueueItem): the item whose priority needs to change. priority (int): the new priority. raise (LookupError): if item not present. """ pos = self._reverse[item] self._queue[pos].priority = priority self._updown_heap(pos) def length(self): """Return the number of elements in the queue. return (int): length of the queue """ return len(self._queue) def empty(self): """Return if the queue is empty. return (bool): is the queue empty? """ return self.length() == 0 def get_status(self): """Return the content of the queue. Note that the order may be not correct, but the first element is the one at the top. return ([QueueEntry]): a list of entries containing the representation of the item, the priority and the timestamp. """ return [{ 'item': entry.item.to_dict(), 'priority': entry.priority, 'timestamp': make_timestamp(entry.timestamp) } for entry in self._queue]
class WorkerPool(object): """This class keeps the state of the workers attached to ES, and allow the ES to get a usable worker when it needs it. """ WORKER_INACTIVE = None WORKER_DISABLED = "disabled" # Seconds after which we declare a worker stale. WORKER_TIMEOUT = timedelta(seconds=600) def __init__(self, service): """service (Service): the EvaluationService using this WorkerPool. """ self._service = service self._worker = {} # These dictionary stores data about the workers (identified # by their shard number). Schedule disabling to True means # that we are going to disable the worker as soon as possible # (when it finishes the current operations). The current # operations are also discarded because we already re-assigned # it. Ignore is true if the next results coming from the # worker should be discarded. Operations is the list of # operations currently executing. Operations to ignore is the # list of operations to ignore in the next batch of results. # Type: {int: [ESOperation]} self._operations = {} # Type: {int: [ESOperation]} self._operations_to_ignore = {} # Type: {int: Datetime|None} self._start_time = {} # Type: {int: bool} self._schedule_disabling = {} # Type: {int: bool} self._ignore = {} # TODO: given the number of pieces data associated to each # worker, this class could be simplified by creating a new # WorkerPoolItem class. # TODO: at the moment race conditions during the periodic # checks cannot be excluded. A refactoring of this class # should take that into account. # A reverse lookup dictionary mapping operations to shards. # Type: {ESOperation: int} self._operations_reverse = dict() # A lock to ensure that the reverse lookup stays in sync with # the operations lists. self._operation_lock = gevent.lock.RLock() # Event set when there are workers available to take jobs. It # is only guaranteed that if a worker is available, then this # event is set. In other words, the fact that this event is # set does not mean that there is a worker available. self._workers_available_event = Event() def __len__(self): return len(self._worker) def __contains__(self, operation): return operation in self._operations_reverse def _remove_operations(self, shard, new_operation): """Safely remove operations from a worker, assigning a new status. shard (int): the worker from which to remove operations. new_operations (unicode|None): the new operation, which can be INACTIVE or DISABLED. """ with self._operation_lock: operations = self._operations[shard] self._operations[shard] = new_operation if isinstance(operations, list): for operation in operations: del self._operations_reverse[operation] def _add_operations(self, shard, operations): """Assigns new operations to a currently inactive worker. shard (int): shard of the worker. operations ([ESOperation]) operations to assign to the worker. """ if self._operations[shard] != WorkerPool.WORKER_INACTIVE: raise ValueError("Shard %s is already doing an operation.", shard) with self._operation_lock: self._operations[shard] = operations for operation in operations: self._operations_reverse[operation] = shard def wait_for_workers(self): """Wait until a worker might be available.""" self._workers_available_event.wait() def add_worker(self, worker_coord): """Add a new worker to the worker pool. worker_coord (ServiceCoord): the coordinates of the worker. """ shard = worker_coord.shard # Instruct GeventLibrary to connect ES to the Worker. self._worker[shard] = self._service.connect_to( worker_coord, on_connect=self.on_worker_connected) # And we fill all data. self._operations[shard] = WorkerPool.WORKER_INACTIVE self._operations_to_ignore[shard] = [] self._start_time[shard] = None self._schedule_disabling[shard] = False self._ignore[shard] = False self._workers_available_event.set() logger.debug("Worker %s added.", shard) def on_worker_connected(self, worker_coord): """To be called when a worker comes alive after being offline. We use this callback to instruct the worker to precache all files concerning the contest. worker_coord (ServiceCoord): the coordinates of the worker that came online. """ shard = worker_coord.shard logger.info("Worker %s online again.", shard) if self._service.contest_id is not None: self._worker[shard].precache_files( contest_id=self._service.contest_id ) # We don't requeue the operation, because a connection lost # does not invalidate a potential result given by the worker # (as the problem was the connection and not the machine on # which the worker is). But the worker could have been idling, # so we wake up the consumers. self._workers_available_event.set() def acquire_worker(self, operations): """Tries to assign an operation to an available worker. If no workers are available then this returns None, otherwise this returns the chosen worker. operations ([ESOperation]): the operations to assign to a worker. return (int|None): None if no workers are available, the worker assigned to the operation otherwise. """ # We look for an available worker. try: shard = self.find_worker(WorkerPool.WORKER_INACTIVE, require_connection=True, random_worker=True) except LookupError: self._workers_available_event.clear() return None # Then we fill the info for future memory. self._add_operations(shard, operations) logger.debug("Worker %s acquired.", shard) self._start_time[shard] = make_datetime() with SessionGen() as session: jobs = [] datasets = {} submissions = {} user_tests = {} for operation in operations: if operation.dataset_id not in datasets: datasets[operation.dataset_id] = Dataset.get_from_id( operation.dataset_id, session) object_ = None if operation.for_submission(): if operation.object_id not in submissions: submissions[operation.object_id] = \ Submission.get_from_id( operation.object_id, session) object_ = submissions[operation.object_id] else: if operation.object_id not in user_tests: user_tests[operation.object_id] = \ UserTest.get_from_id(operation.object_id, session) object_ = user_tests[operation.object_id] logger.info("Asking worker %s to `%s'.", shard, operation) jobs.append(Job.from_operation( operation, object_, datasets[operation.dataset_id])) job_group_dict = JobGroup(jobs).export_to_dict() self._worker[shard].execute_job_group( job_group_dict=job_group_dict, callback=self._service.action_finished, plus=shard) return shard def release_worker(self, shard): """To be called by ES when it receives a notification that an operation finished. Note: if the worker is scheduled to be disabled, then we disable it, and notify the ES to discard the outcome obtained by the worker. shard (int): the worker to release. return (bool|[ESOperation]): if boolean, whether the result is to be ignored; if a list, the list of operation for which the results should be ignored. """ if self._operations[shard] == WorkerPool.WORKER_INACTIVE: err_msg = "Trying to release worker while it's inactive." logger.error(err_msg) raise ValueError(err_msg) # If the worker has already been disabled, ignore the result # and keep the worker disabled. if self._operations[shard] == WorkerPool.WORKER_DISABLED: return True ret = self._ignore[shard] with self._operation_lock: to_ignore = self._operations_to_ignore[shard] self._operations_to_ignore[shard] = [] self._start_time[shard] = None self._ignore[shard] = False if self._schedule_disabling[shard]: self._remove_operations(shard, WorkerPool.WORKER_DISABLED) self._schedule_disabling[shard] = False logger.info("Worker %s released and disabled.", shard) else: self._remove_operations(shard, WorkerPool.WORKER_INACTIVE) self._workers_available_event.set() logger.debug("Worker %s released.", shard) if ret is False and to_ignore != []: return to_ignore else: return ret def find_worker(self, operation, require_connection=False, random_worker=False): """Return a worker whose assigned operation is operation. Remember that there is a placeholder operation to signal that the worker is not doing anything (or disabled). operation (ESOperation|unicode|None): the operation we are looking for, or WorkerPool.WORKER_*. require_connection (bool): True if we want to find a worker doing the operation and that is actually connected to us (i.e., did not die). random_worker (bool): if True, choose uniformly amongst all workers doing the operation. returns (int): the shard of a worker working on operation. raise (LookupError): if nothing has been found. """ pool = [] for shard, worker_operation in iteritems(self._operations): if worker_operation == operation: if not require_connection or self._worker[shard].connected: pool.append(shard) if not random_worker: return shard if pool == []: raise LookupError("No such operation.") else: return random.choice(pool) def ignore_operation(self, operation): """Mark the operation to be ignored. operation (ESOperation): the operation to ignore. raise (LookupError): if operation is not found. """ try: with self._operation_lock: shard = self._operations_reverse[operation] self._operations_to_ignore[shard].append(operation) except LookupError: logger.debug("Asked to ignore operation `%s' " "that cannot be found.", operation) raise def get_status(self): """Returns a dict with info about the current status of all workers. return (dict): dict of info: current operation, starting time, number of errors, and additional data specified in the operation. """ result = dict() for shard in iterkeys(self._worker): s_time = self._start_time[shard] s_time = make_timestamp(s_time) if s_time is not None else None result["%d" % shard] = { 'connected': self._worker[shard].connected, 'operations': [operation.to_dict() for operation in self._operations[shard]] if isinstance(self._operations[shard], list) else self._operations[shard], 'start_time': s_time} return result def check_timeouts(self): """Check if some worker is not responding in too much time. If this is the case, the worker is scheduled for disabling, and we send it a message trying to shut it down. return ([ESOperation]): list of operations assigned to worker that timeout. """ now = make_datetime() lost_operations = [] for shard in self._worker: if self._start_time[shard] is not None: active_for = now - self._start_time[shard] if active_for > WorkerPool.WORKER_TIMEOUT: # Here shard is a working worker with no sign of # intelligent life for too much time. logger.error("Disabling and shutting down " "worker %d because of no response " "in %s.", shard, active_for) is_busy = (self._operations[shard] != WorkerPool.WORKER_INACTIVE and self._operations[shard] != WorkerPool.WORKER_DISABLED) assert is_busy # We return the operation so ES can do what it needs. if not self._ignore[shard] and \ isinstance(self._operations[shard], list): for operation in self._operations[shard]: if operation not in \ self._operations_to_ignore[shard]: lost_operations.append(operation) # Also, we are not trusting it, so we are not # assigning it new operations even if it comes back to # life. self._schedule_disabling[shard] = True self._ignore[shard] = True self.release_worker(shard) self._worker[shard].quit( reason="No response in %s." % active_for) return lost_operations def disable_worker(self, shard): """Disable a worker. shard (int): which worker to disable. return ([ESOperation]): list of non-ignored operations assigned to the worker. raise (ValueError): if worker is already disabled. """ if self._operations[shard] == WorkerPool.WORKER_DISABLED: err_msg = \ "Trying to disable already disabled worker %s." % shard logger.warning(err_msg) raise ValueError(err_msg) lost_operations = [] if self._operations[shard] == WorkerPool.WORKER_INACTIVE: self._operations[shard] = WorkerPool.WORKER_DISABLED else: # We return all non-ignored operations so ES can do what # it needs. if not self._ignore[shard]: to_ignore = self._operations_to_ignore[shard] if isinstance(self._operations[shard], list): for operation in self._operations[shard]: if operation not in to_ignore: lost_operations.append(operation) # And we mark the worker as disabled (until another action # is taken). self._schedule_disabling[shard] = True self._operations_to_ignore[shard] = [] self._ignore[shard] = True self.release_worker(shard) logger.info("Worker %s disabled.", shard) return lost_operations def enable_worker(self, shard): """Enable a worker that previously was disabled. shard (int): which worker to enable. raise (ValueError): if worker is not disabled. """ if self._operations[shard] != WorkerPool.WORKER_DISABLED: err_msg = \ "Trying to enable worker %s which is not disabled." % shard logger.error(err_msg) raise ValueError(err_msg) self._operations[shard] = WorkerPool.WORKER_INACTIVE self._operations_to_ignore[shard] = [] self._workers_available_event.set() logger.info("Worker %s enabled.", shard) def check_connections(self): """Check if a worker we assigned an operation to disconnects. In this case, requeue the operation. return ([ESOperation]): list of operations assigned to worker that disconnected. """ lost_operations = [] for shard in self._worker: if not self._worker[shard].connected and \ self._operations[shard] not in [ WorkerPool.WORKER_DISABLED, WorkerPool.WORKER_INACTIVE]: if not self._ignore[shard]: lost_operations += self._operations[shard] self.release_worker(shard) return lost_operations
def __init__( self, chain: BlockChainService, query_start_block: typing.BlockNumber, default_registry: TokenNetworkRegistry, default_secret_registry: SecretRegistry, private_key_bin, transport, config, discovery=None, ): if not isinstance(private_key_bin, bytes) or len(private_key_bin) != 32: raise ValueError('invalid private_key') self.tokennetworkids_to_connectionmanagers = dict() self.identifier_to_results: typing.Dict[typing.PaymentIdentifier, AsyncResult, ] = dict() self.chain: BlockChainService = chain self.default_registry = default_registry self.query_start_block = query_start_block self.default_secret_registry = default_secret_registry self.config = config self.privkey = private_key_bin self.address = privatekey_to_address(private_key_bin) self.discovery = discovery self.private_key = PrivateKey(private_key_bin) self.pubkey = self.private_key.public_key.format(compressed=False) self.transport = transport self.blockchain_events = BlockchainEvents() self.alarm = AlarmTask(chain) self.shutdown_timeout = config['shutdown_timeout'] self.stop_event = Event() self.start_event = Event() self.chain.client.inject_stop_event(self.stop_event) self.wal = None self.snapshot_group = 0 # This flag will be used to prevent the service from processing # state changes events until we know that pending transactions # have been dispatched. self.dispatch_events_lock = Semaphore(1) self.database_path = config['database_path'] if self.database_path != ':memory:': database_dir = os.path.dirname(config['database_path']) os.makedirs(database_dir, exist_ok=True) self.database_dir = database_dir # Prevent concurrent access to the same db self.lock_file = os.path.join(self.database_dir, '.lock') self.db_lock = filelock.FileLock(self.lock_file) else: self.database_path = ':memory:' self.database_dir = None self.lock_file = None self.serialization_file = None self.db_lock = None self.event_poll_lock = gevent.lock.Semaphore()
class EchoNode: # pragma: no unittest def __init__(self, api: RaidenAPI, token_address: TokenAddress): assert isinstance(api, RaidenAPI) self.ready = Event() self.api = api self.token_address = token_address existing_channels = self.api.get_channel_list( api.raiden.default_registry.address, self.token_address) open_channels = [ channel_state for channel_state in existing_channels if channel.get_status(channel_state) == ChannelState.STATE_OPENED ] if len(open_channels) == 0: token_proxy = self.api.raiden.proxy_manager.token( self.token_address) if not token_proxy.balance_of(self.api.raiden.address) > 0: raise ValueError( f"Not enough funds for echo node " f"{to_checksum_address(self.api.raiden.address)} for token " f"{to_checksum_address(self.token_address)}") # Using the balance of the node as funds funds = TokenAmount(token_proxy.balance_of( self.api.raiden.address)) self.api.token_network_connect( registry_address=self.api.raiden.default_registry.address, token_address=self.token_address, funds=funds, initial_channel_target=10, joinable_funds_target=0.5, ) self.num_seen_events = 0 self.received_transfers: Queue[EventPaymentReceivedSuccess] = Queue() self.stop_signal = None # used to signal REMOVE_CALLBACK and stop echo_workers self.greenlets: Set[Greenlet] = set() self.lock = BoundedSemaphore() self.seen_transfers: Deque[EventPaymentReceivedSuccess] = deque( list(), TRANSFER_MEMORY) self.num_handled_transfers = 0 self.lottery_pool = Queue() # register ourselves with the raiden alarm task self.api.raiden.alarm.register_callback(self.echo_node_alarm_callback) self.echo_worker_greenlet = gevent.spawn(self.echo_worker) log.info("Echo node started") def echo_node_alarm_callback(self, block: Dict[str, Any]): """ This can be registered with the raiden AlarmTask. If `EchoNode.stop()` is called, it will give the return signal to be removed from the AlarmTask callbacks. """ if not self.ready.is_set(): self.ready.set() log.debug( "echo_node callback", node=to_checksum_address(self.api.address), block_number=block["number"], ) if self.stop_signal is not None: return REMOVE_CALLBACK else: self.greenlets.add(gevent.spawn(self.poll_all_received_events)) return True def poll_all_received_events(self) -> None: """ This will be triggered once for each `echo_node_alarm_callback`. It polls all channels for `EventPaymentReceivedSuccess` events, adds all new events to the `self.received_transfers` queue and respawns `self.echo_worker`, if it died. """ locked = False try: with Timeout(10): locked = self.lock.acquire(blocking=False) if not locked: return else: received_transfers: List[ Event] = self.api.get_raiden_events_payment_history( token_address=self.token_address, offset=self.num_seen_events) received_transfers = [ event for event in received_transfers if type(event) == EventPaymentReceivedSuccess ] for event in received_transfers: transfer = copy.deepcopy(event) self.received_transfers.put(transfer) # set last_poll_block after events are enqueued (timeout safe) if received_transfers: self.num_seen_events += len(received_transfers) if not bool(self.echo_worker_greenlet): log.debug( "Restarting echo_worker_greenlet", node=to_checksum_address(self.api.address), dead=self.echo_worker_greenlet.dead, successful=self.echo_worker_greenlet.successful(), exception=self.echo_worker_greenlet.exception, ) self.echo_worker_greenlet = gevent.spawn( self.echo_worker) except Timeout: log.info("Timeout while polling for events") finally: if locked: self.lock.release() def echo_worker(self): """ The `echo_worker` works through the `self.received_transfers` queue and spawns `self.on_transfer` greenlets for all not-yet-seen transfers. """ log.debug("echo worker", qsize=self.received_transfers.qsize()) while self.stop_signal is None: if self.received_transfers.qsize() > 0: transfer = self.received_transfers.get() if transfer in self.seen_transfers: log.debug( "Duplicate transfer ignored", node=to_checksum_address(self.api.address), initiator=to_checksum_address(transfer.initiator), amount=transfer.amount, identifier=transfer.identifier, ) else: self.seen_transfers.append(transfer) self.greenlets.add(gevent.spawn(self.on_transfer, transfer)) else: gevent.sleep(0.5) def on_transfer(self, transfer): """ This handles the echo logic, as described in https://github.com/raiden-network/raiden/issues/651: - for transfers with an amount that satisfies `amount % 3 == 0`, it sends a transfer with an amount of `amount - 1` back to the initiator - for transfers with a "lucky number" amount `amount == 7` it does not send anything back immediately -- after having received "lucky number transfers" from 7 different addresses it sends a transfer with `amount = 49` to one randomly chosen one (from the 7 lucky addresses) - consecutive entries to the lucky lottery will receive the current pool size as the `echo_amount` - for all other transfers it sends a transfer with the same `amount` back to the initiator """ echo_amount = 0 if transfer.amount % 3 == 0: log.info( "Received amount divisible by three", node=to_checksum_address(self.api.address), initiator=to_checksum_address(transfer.initiator), amount=transfer.amount, identifier=transfer.identifier, ) echo_amount = TokenAmount(transfer.amount - 1) elif transfer.amount == 7: log.info( "Received lottery entry", node=to_checksum_address(self.api.address), initiator=to_checksum_address(transfer.initiator), amount=transfer.amount, identifier=transfer.identifier, poolsize=self.lottery_pool.qsize(), ) # obtain a local copy of the pool pool = self.lottery_pool.copy() tickets = [pool.get() for _ in range(pool.qsize())] assert pool.empty() del pool if any(ticket.initiator == transfer.initiator for ticket in tickets): assert transfer not in tickets log.debug( "Duplicate lottery entry", node=to_checksum_address(self.api.address), initiator=to_checksum_address(transfer.initiator), identifier=transfer.identifier, poolsize=len(tickets), ) # signal the poolsize to the participant echo_amount = len(tickets) # payout elif len(tickets) == 6: log.info("Payout!") # reset the pool assert self.lottery_pool.qsize() == 6 self.lottery_pool = Queue() # add the new participant tickets.append(transfer) # choose the winner transfer = random.choice(tickets) echo_amount = 49 else: self.lottery_pool.put(transfer) else: log.debug( "Received transfer", node=to_checksum_address(self.api.address), initiator=to_checksum_address(transfer.initiator), amount=transfer.amount, identifier=transfer.identifier, ) echo_amount = transfer.amount if echo_amount: echo_identifier = transfer.identifier + echo_amount log.debug( "Sending echo transfer", node=to_checksum_address(self.api.address), target=to_checksum_address(transfer.initiator), amount=echo_amount, original_identifier=transfer.identifier, echo_identifier=echo_identifier, token_address=to_checksum_address(self.token_address), num_handled_transfers=self.num_handled_transfers + 1, ) self.api.transfer( registry_address=self.api.raiden.default_registry.address, token_address=self.token_address, amount=echo_amount, target=transfer.initiator, identifier=echo_identifier, ) self.num_handled_transfers += 1 def stop(self): self.stop_signal = True self.greenlets.add(self.echo_worker_greenlet) gevent.joinall(self.greenlets, raise_error=True)
import time import random from flask import Blueprint, Response from apps import AppBlueprint from threading import Thread from gevent.event import Event, AsyncResult from gevent import sleep blueprint = AppBlueprint(blueprint=Blueprint('HelloWorldPage', __name__)) blueprint2 = AppBlueprint(blueprint=Blueprint('HelloWorldPage2', __name__), rule='/<string:action>') __sync_signal = Event() random_event_result = AsyncResult() def load(*args, **kwargs): return {} # These blueprints will be registered with the Flask app and can be used to make your own endpoints. @blueprint.blueprint.route('/test_blueprint') def test_basic_blueprint(): # This can be called using the url /apps/HelloWorld/test_blueprint return 'successfully called basic blueprint' def random_number_receiver(): while True: data = random_event_result.get() yield 'data: %s\n\n' % data
def single_queue_send( transport: 'UDPTransport', recipient: Address, queue: Queue_T, queue_identifier: QueueIdentifier, event_stop: Event, event_healthy: Event, event_unhealthy: Event, message_retries: int, message_retry_timeout: int, message_retry_max_timeout: int, ): """ Handles a single message queue for `recipient`. Notes: - This task must be the only consumer of queue. - This task can be killed at any time, but the intended usage is to stop it with the event_stop. - If there are many queues for the same recipient, it is the caller's responsibility to not start them together to avoid congestion. - This task assumes the endpoint is never cleared after it's first known. If this assumption changes the code must be updated to handle unknown addresses. """ # A NotifyingQueue is required to implement cancelability, otherwise the # task cannot be stopped while the greenlet waits for an element to be # inserted in the queue. if not isinstance(queue, NotifyingQueue): raise ValueError('queue must be a NotifyingQueue.') # Reusing the event, clear must be carefully done data_or_stop = event_first_of( queue, event_stop, ) # Wait for the endpoint registration or to quit transport.log.debug( 'queue: waiting for node to become healthy', queue_identifier=queue_identifier, queue_size=len(queue), ) event_first_of( event_healthy, event_stop, ).wait() transport.log.debug( 'queue: processing queue', queue_identifier=queue_identifier, queue_size=len(queue), ) while True: data_or_stop.wait() if event_stop.is_set(): transport.log.debug( 'queue: stopping', queue_identifier=queue_identifier, queue_size=len(queue), ) return # The queue is not empty at this point, so this won't raise Empty. # This task being the only consumer is a requirement. (messagedata, message_id) = queue.peek(block=False) transport.log.debug( 'queue: sending message', recipient=pex(recipient), msgid=message_id, queue_identifier=queue_identifier, queue_size=len(queue), ) backoff = timeout_exponential_backoff( message_retries, message_retry_timeout, message_retry_max_timeout, ) acknowledged = retry_with_recovery( transport, messagedata, message_id, recipient, event_stop, event_healthy, event_unhealthy, backoff, ) if acknowledged: queue.get() # Checking the length of the queue does not trigger a # context-switch, so it's safe to assume the length of the queue # won't change under our feet and when a new item will be added the # event will be set again. if not queue: data_or_stop.clear() if event_stop.is_set(): return
class RaidenService: """ A Raiden node. """ def __init__( self, chain: BlockChainService, query_start_block: typing.BlockNumber, default_registry: TokenNetworkRegistry, default_secret_registry: SecretRegistry, private_key_bin, transport, config, discovery=None, ): if not isinstance(private_key_bin, bytes) or len(private_key_bin) != 32: raise ValueError('invalid private_key') self.tokennetworkids_to_connectionmanagers = dict() self.identifier_to_results: typing.Dict[typing.PaymentIdentifier, AsyncResult, ] = dict() self.chain: BlockChainService = chain self.default_registry = default_registry self.query_start_block = query_start_block self.default_secret_registry = default_secret_registry self.config = config self.privkey = private_key_bin self.address = privatekey_to_address(private_key_bin) self.discovery = discovery self.private_key = PrivateKey(private_key_bin) self.pubkey = self.private_key.public_key.format(compressed=False) self.transport = transport self.blockchain_events = BlockchainEvents() self.alarm = AlarmTask(chain) self.shutdown_timeout = config['shutdown_timeout'] self.stop_event = Event() self.start_event = Event() self.chain.client.inject_stop_event(self.stop_event) self.wal = None self.snapshot_group = 0 # This flag will be used to prevent the service from processing # state changes events until we know that pending transactions # have been dispatched. self.dispatch_events_lock = Semaphore(1) self.database_path = config['database_path'] if self.database_path != ':memory:': database_dir = os.path.dirname(config['database_path']) os.makedirs(database_dir, exist_ok=True) self.database_dir = database_dir # Prevent concurrent access to the same db self.lock_file = os.path.join(self.database_dir, '.lock') self.db_lock = filelock.FileLock(self.lock_file) else: self.database_path = ':memory:' self.database_dir = None self.lock_file = None self.serialization_file = None self.db_lock = None self.event_poll_lock = gevent.lock.Semaphore() def start_async(self) -> Event: """ Start the node asynchronously. """ self.start_event.clear() self.stop_event.clear() if self.database_dir is not None: self.db_lock.acquire(timeout=0) assert self.db_lock.is_locked # start the registration early to speed up the start if self.config['transport_type'] == 'udp': endpoint_registration_greenlet = gevent.spawn( self.discovery.register, self.address, self.config['transport']['udp']['external_ip'], self.config['transport']['udp']['external_port'], ) # The database may be :memory: storage = sqlite.SQLiteStorage(self.database_path, serialize.PickleSerializer()) self.wal = wal.restore_from_latest_snapshot( node.state_transition, storage, ) if self.wal.state_manager.current_state is None: log.debug('No recoverable state available, created inital state') block_number = self.chain.block_number() state_change = ActionInitChain( random.Random(), block_number, self.chain.node_address, self.chain.network_id, ) self.wal.log_and_dispatch(state_change, block_number) payment_network = PaymentNetworkState( self.default_registry.address, [], # empty list of token network states as it's the node's startup ) state_change = ContractReceiveNewPaymentNetwork( constants.NULL_ADDRESS, payment_network, ) self.handle_state_change(state_change) # On first run Raiden needs to fetch all events for the payment # network, to reconstruct all token network graphs and find opened # channels last_log_block_number = 0 else: # The `Block` state change is dispatched only after all the events # for that given block have been processed, filters can be safely # installed starting from this position without losing events. last_log_block_number = views.block_number( self.wal.state_manager.current_state) log.debug('Restored state from WAL', last_restored_block=last_log_block_number) # Restore the current snapshot group self.snapshot_group = last_log_block_number // SNAPSHOT_BLOCK_COUNT # Install the filters using the correct from_block value, otherwise # blockchain logs can be lost. self.install_all_blockchain_filters( self.default_registry, self.default_secret_registry, last_log_block_number, ) # Complete the first_run of the alarm task and synchronize with the # blockchain since the last run. # # Notes about setup order: # - The filters must be polled after the node state has been primed, # otherwise the state changes won't have effect. # - The alarm must complete its first run before the transport is started, # to avoid rejecting messages for unknown channels. self.alarm.register_callback(self._callback_new_block) self.alarm.first_run() chain_state = views.state_from_raiden(self) # Dispatch pending transactions pending_transactions = views.get_pending_transactions(chain_state, ) log.debug( 'Processing pending transactions', num_pending_transactions=len(pending_transactions), ) with self.dispatch_events_lock: for transaction in pending_transactions: on_raiden_event(self, transaction) self.alarm.start() queueids_to_queues = views.get_all_messagequeues(chain_state) self.transport.start(self, queueids_to_queues) # Health check needs the transport layer self.start_neighbours_healthcheck() if self.config['transport_type'] == 'udp': def set_start_on_registration(_): self.start_event.set() endpoint_registration_greenlet.link(set_start_on_registration) else: self.start_event.set() return self.start_event def start(self) -> Event: """ Start the node. """ self.start_async().wait() def start_neighbours_healthcheck(self): for neighbour in views.all_neighbour_nodes( self.wal.state_manager.current_state): if neighbour != ConnectionManager.BOOTSTRAP_ADDR: self.start_health_check_for(neighbour) def stop(self): """ Stop the node. """ # Needs to come before any greenlets joining self.stop_event.set() self.transport.stop_and_wait() self.alarm.stop_async() wait_for = [self.alarm] wait_for.extend(getattr(self.transport, 'greenlets', [])) # We need a timeout to prevent an endless loop from trying to # contact the disconnected client gevent.wait(wait_for, timeout=self.shutdown_timeout) # Filters must be uninstalled after the alarm task has stopped. Since # the events are polled by an alarm task callback, if the filters are # uninstalled before the alarm task is fully stopped the callback # `poll_blockchain_events` will fail. # # We need a timeout to prevent an endless loop from trying to # contact the disconnected client try: with gevent.Timeout(self.shutdown_timeout): self.blockchain_events.uninstall_all_event_listeners() except (gevent.timeout.Timeout, RaidenShuttingDown): pass self.blockchain_events.reset() if self.db_lock is not None: self.db_lock.release() def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, pex(self.address)) def get_block_number(self): return views.block_number(self.wal.state_manager.current_state) def handle_state_change(self, state_change, block_number=None): log.debug('STATE CHANGE', node=pex(self.address), state_change=state_change) if block_number is None: block_number = self.get_block_number() # Take a snapshot every SNAPSHOT_BLOCK_COUNT # TODO: Gather more data about storage requirements # and update the value to specify how often we need # capturing a snapshot should take place new_snapshot_group = block_number // SNAPSHOT_BLOCK_COUNT if new_snapshot_group > self.snapshot_group: log.debug(f'Storing snapshot at block: {block_number}') self.wal.snapshot() self.snapshot_group = new_snapshot_group event_list = self.wal.log_and_dispatch(state_change, block_number) if self.dispatch_events_lock.locked(): return [] for event in event_list: log.debug('RAIDEN EVENT', node=pex(self.address), raiden_event=event) on_raiden_event(self, event) return event_list def set_node_network_state(self, node_address, network_state): state_change = ActionChangeNodeNetworkState(node_address, network_state) self.wal.log_and_dispatch(state_change, self.get_block_number()) def start_health_check_for(self, node_address): self.transport.start_health_check(node_address) def _callback_new_block(self, current_block_number, chain_id): """Called once a new block is detected by the alarm task. Note: This should be called only once per block, otherwise there will be duplicated `Block` state changes in the log. Therefore this method should be called only once a new block is mined with the appropriate block_number argument from the AlarmTask. """ # Raiden relies on blockchain events to update its off-chain state, # therefore some APIs /used/ to forcefully poll for events. # # This was done for APIs which have on-chain side-effects, e.g. # openning a channel, where polling the event is required to update # off-chain state to providing a consistent view to the caller, e.g. # the channel exists after the API call returns. # # That pattern introduced a race, because the events are returned only # once per filter, and this method would be called concurrently by the # API and the AlarmTask. The following lock is necessary, to ensure the # expected side-effects are properly applied (introduced by the commit # 3686b3275ff7c0b669a6d5e2b34109c3bdf1921d) with self.event_poll_lock: for event in self.blockchain_events.poll_blockchain_events( current_block_number): # These state changes will be procesed with a block_number # which is /larger/ than the ChainState's block_number. on_blockchain_event(self, event, current_block_number, chain_id) # On restart the Raiden node will re-create the filters with the # ethereum node. These filters will have the from_block set to the # value of the latest Block state change. To avoid missing events # the Block state change is dispatched only after all of the events # have been processed. # # This means on some corner cases a few events may be applied # twice, this will happen if the node crashed and some events have # been processed but the Block state change has not been # dispatched. state_change = Block(current_block_number) self.handle_state_change(state_change, current_block_number) def sign(self, message): """ Sign message inplace. """ if not isinstance(message, SignedMessage): raise ValueError('{} is not signable.'.format(repr(message))) message.sign(self.private_key) def install_all_blockchain_filters( self, token_network_registry_proxy: TokenNetworkRegistry, secret_registry_proxy: SecretRegistry, from_block: typing.BlockNumber, ): with self.event_poll_lock: node_state = views.state_from_raiden(self) channels = views.list_all_channelstate(node_state) token_networks = views.get_token_network_identifiers( node_state, token_network_registry_proxy.address, ) self.blockchain_events.add_token_network_registry_listener( token_network_registry_proxy, from_block, ) self.blockchain_events.add_secret_registry_listener( secret_registry_proxy, from_block, ) for token_network in token_networks: token_network_proxy = self.chain.token_network(token_network) self.blockchain_events.add_token_network_listener( token_network_proxy, from_block, ) for channel_state in channels: channel_proxy = self.chain.payment_channel( channel_state.token_network_identifier, channel_state.identifier, ) self.blockchain_events.add_payment_channel_listener( channel_proxy, from_block, ) def connection_manager_for_token_network(self, token_network_identifier): if not is_binary_address(token_network_identifier): raise InvalidAddress('token address is not valid.') known_token_networks = views.get_token_network_identifiers( views.state_from_raiden(self), self.default_registry.address, ) if token_network_identifier not in known_token_networks: raise InvalidAddress('token is not registered.') manager = self.tokennetworkids_to_connectionmanagers.get( token_network_identifier) if manager is None: manager = ConnectionManager(self, token_network_identifier) self.tokennetworkids_to_connectionmanagers[ token_network_identifier] = manager return manager def leave_all_token_networks(self): state_change = ActionLeaveAllNetworks() self.wal.log_and_dispatch(state_change, self.get_block_number()) def close_and_settle(self): log.info('raiden will close and settle all channels now') self.leave_all_token_networks() connection_managers = [ cm for cm in self.tokennetworkids_to_connectionmanagers.values() ] if connection_managers: waiting.wait_for_settle_all_channels( self, self.alarm.sleep_time, ) def mediated_transfer_async( self, token_network_identifier, amount, target, identifier, ): """ Transfer `amount` between this node and `target`. This method will start an asynchronous transfer, the transfer might fail or succeed depending on a couple of factors: - Existence of a path that can be used, through the usage of direct or intermediary channels. - Network speed, making the transfer sufficiently fast so it doesn't expire. """ async_result = self.start_mediated_transfer( token_network_identifier, amount, target, identifier, ) return async_result def direct_transfer_async(self, token_network_identifier, amount, target, identifier): """ Do a direct transfer with target. Direct transfers are non cancellable and non expirable, since these transfers are a signed balance proof with the transferred amount incremented. Because the transfer is non cancellable, there is a level of trust with the target. After the message is sent the target is effectively paid and then it is not possible to revert. The async result will be set to False iff there is no direct channel with the target or the payer does not have balance to complete the transfer, otherwise because the transfer is non expirable the async result *will never be set to False* and if the message is sent it will hang until the target node acknowledge the message. This transfer should be used as an optimization, since only two packets are required to complete the transfer (from the payers perspective), whereas the mediated transfer requires 6 messages. """ self.start_health_check_for(target) if identifier is None: identifier = create_default_identifier() direct_transfer = ActionTransferDirect( token_network_identifier, target, identifier, amount, ) self.handle_state_change(direct_transfer) def start_mediated_transfer( self, token_network_identifier, amount, target, identifier, ): self.start_health_check_for(target) if identifier is None: identifier = create_default_identifier() if identifier in self.identifier_to_results: return self.identifier_to_results[identifier] async_result = AsyncResult() self.identifier_to_results[identifier] = async_result secret = random_secret() init_initiator_statechange = initiator_init( self, identifier, amount, secret, token_network_identifier, target, ) # Dispatch the state change even if there are no routes to create the # wal entry. self.handle_state_change(init_initiator_statechange) return async_result def mediate_mediated_transfer(self, transfer: LockedTransfer): init_mediator_statechange = mediator_init(self, transfer) self.handle_state_change(init_mediator_statechange) def target_mediated_transfer(self, transfer: LockedTransfer): self.start_health_check_for(transfer.initiator) init_target_statechange = target_init(transfer) self.handle_state_change(init_target_statechange)
class Worker(Greenlet): """ This class actually performs all of the grunt work, a worker is a Greenlet or Thread which are spawned based on the number of connections defined. """ def __init__(self, connection, work_queue, work_tracker): Greenlet.__init__(self, run=None) # Store NNTPConnection Object self._connection = connection # Our only communication to the outside world # event is a simple event handler of type Event() # we only unblock if this is set self._event = Event() # We always process as many queue entries as we can # TODO: This should contain some sort of container object that # allows us to process many Message-ID, but we need to know # their group they belong into. We also need their expected # filename. We want to allow the NZBFiles to force a filename # optionally, otherwise the yenc= is used instead. A minimum # of those 3 entries `must` be in this input queue. Perhaps # we should just use NNTPContent entries? (they don't have # an article ID though) self._work_queue = work_queue # Store our work tracker self._work_tracker = work_tracker # our exit flag, it is set externally self._exit = Event() def _run(self): """ Read from the work_queue, process it using an NNTPRequest object. """ # block until we have an event to handle. # print "Worker %s ready!" % self while not self._exit.is_set(): # Begin our loop try: request = self._work_queue.get() if request is StopIteration: # during a close() call (defined below) we force # a StopIteration into the queue to force an exit # from a program level return if request.is_set(): # Process has been aborted or is no longer needed continue except StopIteration: # Got Exit return except EmptyQueueException: # Nothing available for us continue # Mark ourselves busy self._work_tracker.mark_busy(self) # If we reach here, we have a request to process request.run(connection=self._connection) # Mark ourselves available again self._work_tracker.mark_available(self) # Ensure our connection is closed before we exit self._connection.close() def __del__(self): # If Ctrl-C is pressed or we're forced to break earlier then we may # end up here. Ensure our connection is closed before we exit self._connection.close()
class CephFSMirrorThrasher(Thrasher, Greenlet): """ CephFSMirrorThrasher:: The CephFSMirrorThrasher thrashes cephfs-mirror daemons during execution of other tasks (workunits, etc). The config is optional. Many of the config parameters are a maximum value to use when selecting a random value from a range. The config is a dict containing some or all of: cluster: [default: ceph] cluster to thrash max_thrash: [default: 1] the maximum number of active cephfs-mirror daemons per cluster will be thrashed at any given time. min_thrash_delay: [default: 60] minimum number of seconds to delay before thrashing again. max_thrash_delay: [default: 120] maximum number of seconds to delay before thrashing again. max_revive_delay: [default: 10] maximum number of seconds to delay before bringing back a thrashed cephfs-mirror daemon. randomize: [default: true] enables randomization and use the max/min values seed: [no default] seed the random number generator Examples:: The following example disables randomization, and uses the max delay values: tasks: - ceph: - cephfs_mirror_thrash: randomize: False max_thrash_delay: 10 """ def __init__(self, ctx, config, cluster, daemons): super(CephFSMirrorThrasher, self).__init__() self.ctx = ctx self.config = config self.cluster = cluster self.daemons = daemons self.logger = log self.name = 'thrasher.cephfs_mirror.[{cluster}]'.format( cluster=cluster) self.stopping = Event() self.randomize = bool(self.config.get('randomize', True)) self.max_thrash = int(self.config.get('max_thrash', 1)) self.min_thrash_delay = float(self.config.get('min_thrash_delay', 5.0)) self.max_thrash_delay = float(self.config.get('max_thrash_delay', 10)) self.max_revive_delay = float(self.config.get('max_revive_delay', 15.0)) def _run(self): try: self.do_thrash() except Exception as e: # See _run exception comment for MDSThrasher self.set_thrasher_exception(e) self.logger.exception("exception:") # Allow successful completion so gevent doesn't see an exception. # The DaemonWatchdog will observe the error and tear down the test. def log(self, x): """Write data to logger assigned to this CephFSMirrorThrasher""" self.logger.info(x) def stop(self): self.stopping.set() def do_thrash(self): """ Perform the random thrashing action """ self.log('starting thrash for cluster {cluster}'.format( cluster=self.cluster)) stats = { "kill": 0, } while not self.stopping.is_set(): delay = self.max_thrash_delay if self.randomize: delay = random.randrange(self.min_thrash_delay, self.max_thrash_delay) if delay > 0.0: self.log('waiting for {delay} secs before thrashing'.format( delay=delay)) self.stopping.wait(delay) if self.stopping.is_set(): continue killed_daemons = [] weight = 1.0 / len(self.daemons) count = 0 for daemon in self.daemons: skip = random.uniform(0.0, 1.0) if weight <= skip: self.log( 'skipping daemon {label} with skip ({skip}) > weight ({weight})' .format(label=daemon.id_, skip=skip, weight=weight)) continue self.log('kill {label}'.format(label=daemon.id_)) try: daemon.signal(signal.SIGTERM) except Exception as e: self.log(f'exception when stopping mirror daemon: {e}') else: killed_daemons.append(daemon) stats['kill'] += 1 # if we've reached max_thrash, we're done count += 1 if count >= self.max_thrash: break if killed_daemons: # wait for a while before restarting delay = self.max_revive_delay if self.randomize: delay = random.randrange(0.0, self.max_revive_delay) self.log( 'waiting for {delay} secs before reviving daemons'.format( delay=delay)) sleep(delay) for daemon in killed_daemons: self.log('waiting for {label}'.format(label=daemon.id_)) try: run.wait([daemon.proc], timeout=600) except CommandFailedError: pass except: self.log( 'Failed to stop {label}'.format(label=daemon.id_)) try: # try to capture a core dump daemon.signal(signal.SIGABRT) except socket.error: pass raise finally: daemon.reset() for daemon in killed_daemons: self.log('reviving {label}'.format(label=daemon.id_)) daemon.start() for stat in stats: self.log("stat['{key}'] = {value}".format(key=stat, value=stats[stat]))
from .utilities import JSONDecodeError, load_json, dump_json, static_routes, \ generate_endpoint from .database import Invite, Message, User, get_invite_by_code, \ get_messages_by_timestamp, get_user_by_name, set_invite_by_code, \ set_message, set_user from typing import Any, Dict, Callable, Union, List, Optional from datetime import datetime as DateTime from base64 import b64decode from re import compile as regex_compile from os import path auth_regex = regex_compile(r"^(?:(\w+) )?(.*)$") token_regex = regex_compile(r"^(\w+):(.*)$") username_regex = regex_compile(r"[a-z_]{2,32}") mut_message_event = Event() def respond_error(job: HTTPJob, message: str, code: Union[str, int] = 400): content = f'{{"message":"{message}"}}'.encode("utf-8") headers = { "Content-Type": "application/json; charset=utf-8", "Content-Length": str(len(content)) } job.write_head(code, headers) job.close_body(content) def get_authorized_user(auth_or_rq: Union[HTTPJob, Optional[str]]): """Gets the authorized user via the request's authorization header. If for any
class UDPTransport(Runnable): UDP_MAX_MESSAGE_SIZE = 1200 log = log log_healthcheck = log_healthcheck def __init__(self, address, discovery, udpsocket, throttle_policy, config): super().__init__() # these values are initialized by the start method self.queueids_to_queues: Dict = dict() self.raiden: RaidenService self.message_handler: MessageHandler self.discovery = discovery self.config = config self.address = address self.retry_interval = config['retry_interval'] self.retries_before_backoff = config['retries_before_backoff'] self.nat_keepalive_retries = config['nat_keepalive_retries'] self.nat_keepalive_timeout = config['nat_keepalive_timeout'] self.nat_invitation_timeout = config['nat_invitation_timeout'] self.event_stop = Event() self.event_stop.set() self.greenlets = list() self.addresses_events = dict() self.messageids_to_asyncresults = dict() # Maps the addresses to a dict with the latest nonce (using a dict # because python integers are immutable) self.nodeaddresses_to_nonces = dict() cache = cachetools.TTLCache( maxsize=50, ttl=CACHE_TTL, ) cache_wrapper = cachetools.cached(cache=cache) self.get_host_port = cache_wrapper(discovery.get) self.throttle_policy = throttle_policy pool = gevent.pool.Pool() self.server = DatagramServer( udpsocket, handle=self.receive, spawn=pool, ) def start( self, raiden_service: RaidenService, message_handler: MessageHandler, prev_auth_data: str, # pylint: disable=unused-argument ): if not self.event_stop.ready(): raise RuntimeError('UDPTransport started while running') self.event_stop.clear() self.raiden = raiden_service self.log = log.bind(node=pex(self.raiden.address)) self.log_healthcheck = log_healthcheck.bind( node=pex(self.raiden.address)) self.message_handler = message_handler # server.stop() clears the handle and the pool. Since this may be a # restart the handle must always be set self.server.set_handle(self.receive) pool = gevent.pool.Pool() self.server.set_spawn(pool) self.server.start() self.log.debug('UDP started') super().start() def _run(self): # pylint: disable=method-hidden """ Runnable main method, perform wait on long-running subtasks """ try: self.event_stop.wait() except gevent.GreenletExit: # killed without exception self.event_stop.set() gevent.killall(self.greenlets) # kill children raise # re-raise to keep killed status except Exception: self.stop() # ensure cleanup and wait on subtasks raise def stop(self): if self.event_stop.ready(): return # double call, happens on normal stop, ignore self.event_stop.set() # Stop handling incoming packets, but don't close the socket. The # socket can only be safely closed after all outgoing tasks are stopped self.server.stop_accepting() # Stop processing the outgoing queues gevent.wait(self.greenlets) # All outgoing tasks are stopped. Now it's safe to close the socket. At # this point there might be some incoming message being processed, # keeping the socket open is not useful for these. self.server.stop() # Calling `.close()` on a gevent socket doesn't actually close the underlying os socket # so we do that ourselves here. # See: https://github.com/gevent/gevent/blob/master/src/gevent/_socket2.py#L208 # and: https://groups.google.com/forum/#!msg/gevent/Ro8lRra3nH0/ZENgEXrr6M0J try: self.server._socket.close() # pylint: disable=protected-access except socket.error: pass # Set all the pending results to False for async_result in self.messageids_to_asyncresults.values(): async_result.set(False) self.log.debug('UDP stopped') del self.log_healthcheck del self.log def get_health_events(self, recipient): """ Starts a healthcheck task for `recipient` and returns a HealthEvents with locks to react on its current state. """ if recipient not in self.addresses_events: self.start_health_check(recipient) return self.addresses_events[recipient] def whitelist(self, address: Address): # pylint: disable=no-self-use,unused-argument """Whitelist peer address to receive communications from This may be called before transport is started, to ensure events generated during start are handled properly. PS: udp currently doesn't do whitelisting, method defined for compatibility with matrix """ return def start_health_check(self, recipient): """ Starts a task for healthchecking `recipient` if there is not one yet. It also whitelists the address """ if recipient not in self.addresses_events: self.whitelist(recipient) # noop for now, for compatibility ping_nonce = self.nodeaddresses_to_nonces.setdefault( recipient, {'nonce': 0}, # HACK: Allows the task to mutate the object ) events = healthcheck.HealthEvents( event_healthy=Event(), event_unhealthy=Event(), ) self.addresses_events[recipient] = events greenlet_healthcheck = gevent.spawn( healthcheck.healthcheck, self, recipient, self.event_stop, events.event_healthy, events.event_unhealthy, self.nat_keepalive_retries, self.nat_keepalive_timeout, self.nat_invitation_timeout, ping_nonce, ) greenlet_healthcheck.name = f'Healthcheck for {pex(recipient)}' greenlet_healthcheck.link_exception(self.on_error) self.greenlets.append(greenlet_healthcheck) def init_queue_for( self, queue_identifier: QueueIdentifier, items: List[QueueItem_T], ) -> NotifyingQueue: """ Create the queue identified by the queue_identifier and initialize it with `items`. """ recipient = queue_identifier.recipient queue = self.queueids_to_queues.get(queue_identifier) assert queue is None queue = NotifyingQueue(items=items) self.queueids_to_queues[queue_identifier] = queue events = self.get_health_events(recipient) greenlet_queue = gevent.spawn( single_queue_send, self, recipient, queue, queue_identifier, self.event_stop, events.event_healthy, events.event_unhealthy, self.retries_before_backoff, self.retry_interval, self.retry_interval * 10, ) if queue_identifier.channel_identifier == CHANNEL_IDENTIFIER_GLOBAL_QUEUE: greenlet_queue.name = f'Queue for {pex(recipient)} - global' else: greenlet_queue.name = ( f'Queue for {pex(recipient)} - {queue_identifier.channel_identifier}' ) greenlet_queue.link_exception(self.on_error) self.greenlets.append(greenlet_queue) self.log.debug( 'new queue created for', queue_identifier=queue_identifier, items_qty=len(items), ) return queue def get_queue_for( self, queue_identifier: QueueIdentifier, ) -> NotifyingQueue: """ Return the queue identified by the given queue identifier. If the queue doesn't exist it will be instantiated. """ queue = self.queueids_to_queues.get(queue_identifier) if queue is None: items: List[QueueItem_T] = list() queue = self.init_queue_for(queue_identifier, items) return queue def send_async( self, queue_identifier: QueueIdentifier, message: Message, ): """ Send a new ordered message to recipient. Messages that use the same `queue_identifier` are ordered. """ recipient = queue_identifier.recipient if not is_binary_address(recipient): raise ValueError('Invalid address {}'.format(pex(recipient))) # These are not protocol messages, but transport specific messages if isinstance(message, (Delivered, Ping, Pong)): raise ValueError('Do not use send for {} messages'.format( message.__class__.__name__)) messagedata = message.encode() if len(messagedata) > self.UDP_MAX_MESSAGE_SIZE: raise ValueError( 'message size exceeds the maximum {}'.format( self.UDP_MAX_MESSAGE_SIZE), ) # message identifiers must be unique message_id = message.message_identifier # ignore duplicates if message_id not in self.messageids_to_asyncresults: self.messageids_to_asyncresults[message_id] = AsyncResult() queue = self.get_queue_for(queue_identifier) queue.put((messagedata, message_id)) assert queue.is_set() self.log.debug( 'Message queued', queue_identifier=queue_identifier, queue_size=len(queue), message=message, ) def send_global( # pylint: disable=unused-argument self, room: str, message: Message, ) -> None: """ This method exists only for interface compatibility with MatrixTransport """ self.log.warning('UDP is unable to send global messages. Ignoring') def maybe_send(self, recipient: Address, message: Message): """ Send message to recipient if the transport is running. """ if not is_binary_address(recipient): raise InvalidAddress('Invalid address {}'.format(pex(recipient))) messagedata = message.encode() host_port = self.get_host_port(recipient) self.maybe_sendraw(host_port, messagedata) def maybe_sendraw_with_result( self, recipient: Address, messagedata: bytes, message_id: UDPMessageID, ) -> AsyncResult: """ Send message to recipient if the transport is running. Returns: An AsyncResult that will be set once the message is delivered. As long as the message has not been acknowledged with a Delivered message the function will return the same AsyncResult. """ async_result = self.messageids_to_asyncresults.get(message_id) if async_result is None: async_result = AsyncResult() self.messageids_to_asyncresults[message_id] = async_result host_port = self.get_host_port(recipient) self.maybe_sendraw(host_port, messagedata) return async_result def maybe_sendraw(self, host_port: Tuple[int, int], messagedata: bytes): """ Send message to recipient if the transport is running. """ # Don't sleep if timeout is zero, otherwise a context-switch is done # and the message is delayed, increasing its latency sleep_timeout = self.throttle_policy.consume(1) if sleep_timeout: gevent.sleep(sleep_timeout) # Check the udp socket is still available before trying to send the # message. There must be *no context-switches after this test*. if hasattr(self.server, 'socket'): self.server.sendto( messagedata, host_port, ) def receive( self, messagedata: bytes, host_port: Tuple[str, int], # pylint: disable=unused-argument ) -> bool: """ Handle an UDP packet. """ # pylint: disable=unidiomatic-typecheck if len(messagedata) > self.UDP_MAX_MESSAGE_SIZE: self.log.warning( 'Invalid message: Packet larger than maximum size', message=encode_hex(messagedata), length=len(messagedata), ) return False try: message = decode(messagedata) except InvalidProtocolMessage as e: self.log.warning( 'Invalid protocol message', error=str(e), message=encode_hex(messagedata), ) return False if type(message) == Pong: assert isinstance(message, Pong), MYPY_ANNOTATION self.receive_pong(message) elif type(message) == Ping: assert isinstance(message, Ping), MYPY_ANNOTATION self.receive_ping(message) elif type(message) == Delivered: assert isinstance(message, Delivered), MYPY_ANNOTATION self.receive_delivered(message) elif message is not None: self.receive_message(message) else: self.log.warning( 'Invalid message: Unknown cmdid', message=encode_hex(messagedata), ) return False return True def receive_message(self, message: Message): """ Handle a Raiden protocol message. The protocol requires durability of the messages. The UDP transport relies on the node's WAL for durability. The message will be converted to a state change, saved to the WAL, and *processed* before the durability is confirmed, which is a stronger property than what is required of any transport. """ self.raiden.on_message(message) # Sending Delivered after the message is decoded and *processed* # gives a stronger guarantee than what is required from a # transport. # # Alternatives are, from weakest to strongest options: # - Just save it on disk and asynchronously process the messages # - Decode it, save to the WAL, and asynchronously process the # state change # - Decode it, save to the WAL, and process it (the current # implementation) delivered_message = Delivered( delivered_message_identifier=message.message_identifier) self.raiden.sign(delivered_message) self.maybe_send( message.sender, delivered_message, ) def receive_delivered(self, delivered: Delivered): """ Handle a Delivered message. The Delivered message is how the UDP transport guarantees persistence by the partner node. The message itself is not part of the raiden protocol, but it's required by this transport to provide the required properties. """ self.raiden.on_message(delivered) message_id = delivered.delivered_message_identifier async_result = self.raiden.transport.messageids_to_asyncresults.get( message_id) # clear the async result, otherwise we have a memory leak if async_result is not None: del self.messageids_to_asyncresults[message_id] async_result.set() else: self.log.warn( 'Unknown delivered message received', message_id=message_id, ) # Pings and Pongs are used to check the health status of another node. They # are /not/ part of the raiden protocol, only part of the UDP transport, # therefore these messages are not forwarded to the message handler. def receive_ping(self, ping: Ping): """ Handle a Ping message by answering with a Pong. """ self.log_healthcheck.debug( 'Ping received', message_id=ping.nonce, message=ping, sender=pex(ping.sender), ) pong = Pong(nonce=ping.nonce) self.raiden.sign(pong) try: self.maybe_send(ping.sender, pong) except (InvalidAddress, UnknownAddress) as e: self.log.debug("Couldn't send the `Delivered` message", e=e) def receive_pong(self, pong: Pong): """ Handles a Pong message. """ message_id = ('ping', pong.nonce, pong.sender) async_result = self.messageids_to_asyncresults.get(message_id) if async_result is not None: self.log_healthcheck.debug( 'Pong received', sender=pex(pong.sender), message_id=pong.nonce, ) async_result.set(True) else: self.log_healthcheck.warn( 'Unknown pong received', message_id=message_id, ) def get_ping(self, nonce: Nonce) -> bytes: """ Returns a signed Ping message. Note: Ping messages don't have an enforced ordering, so a Ping message with a higher nonce may be acknowledged first. """ message = Ping( nonce=nonce, current_protocol_version=constants.PROTOCOL_VERSION, ) self.raiden.sign(message) return message.encode() def set_node_network_state(self, node_address: Address, node_state): state_change = ActionChangeNodeNetworkState(node_address, node_state) self.raiden.handle_and_track_state_change(state_change)
class Queue(gqueue.Queue): '''A subclass of gevent.queue.Queue used to organize communication messaging between Compysition Actors. Parameters: name (str): | The name of this queue. Used in certain actors to determine origin faster than reverse key-value lookup ''' def __init__(self, name, *args, **kwargs): super(Queue, self).__init__(*args, **kwargs) self.name = name self.__has_content = Event() self.__has_content.clear() def get(self, block=False, *args, **kwargs): '''Gets an element from the queue.''' try: element = super(Queue, self).get(block=block, *args, **kwargs) except gqueue.Empty: self.__has_content.clear() raise QueueEmpty("Queue {0} has no waiting events".format( self.name)) if self.qsize() == 0: self.__has_content.clear() return element def put(self, element, *args, **kwargs): '''Puts element in queue.''' try: super(Queue, self).put(element, *args, **kwargs) self.__has_content.set() except gqueue.Full: #only if block = False or (block = True and timeout not None) raise QueueFull(message="Queue {0} is full".format(self.name), queue=self) def wait_until_content(self): '''Blocks until at least 1 slot is taken.''' self.__has_content.wait() def wait_until_empty(self): '''Blocks until the queue is completely empty.''' while not self.__has_content.is_set(): sleep(0) def wait_until_free(self): '''Blocks until the queue has at lease 1 free slot.''' while self.qsize() >= self.maxsize: sleep(0) def dump(self, other_queue): """**Dump all items on this queue to another queue**""" try: while True: other_queue.put(self.get(block=False)) except QueueEmpty: pass
def __init__(self, maxsize=None): from gevent.event import Event Queue.__init__(self, maxsize) self.unfinished_tasks = 0 self._cond = Event() self._cond.set()
class RaidenService: """ A Raiden node. """ def __init__( self, chain: BlockChainService, default_registry: Registry, default_secret_registry: SecretRegistry, private_key_bin, transport, config, discovery=None, ): if not isinstance(private_key_bin, bytes) or len(private_key_bin) != 32: raise ValueError('invalid private_key') invalid_timeout = ( config['settle_timeout'] < NETTINGCHANNEL_SETTLE_TIMEOUT_MIN or config['settle_timeout'] > NETTINGCHANNEL_SETTLE_TIMEOUT_MAX) if invalid_timeout: raise ValueError('settle_timeout must be in range [{}, {}]'.format( NETTINGCHANNEL_SETTLE_TIMEOUT_MIN, NETTINGCHANNEL_SETTLE_TIMEOUT_MAX, )) self.tokens_to_connectionmanagers = dict() self.identifier_to_results = defaultdict(list) self.chain: BlockChainService = chain self.default_registry = default_registry self.default_secret_registry = default_secret_registry self.config = config self.privkey = private_key_bin self.address = privatekey_to_address(private_key_bin) if config['transport_type'] == 'udp': endpoint_registration_event = gevent.spawn( discovery.register, self.address, config['external_ip'], config['external_port'], ) endpoint_registration_event.link_exception( endpoint_registry_exception_handler) self.private_key = PrivateKey(private_key_bin) self.pubkey = self.private_key.public_key.format(compressed=False) self.transport = transport self.blockchain_events = BlockchainEvents() self.alarm = AlarmTask(chain) self.shutdown_timeout = config['shutdown_timeout'] self.stop_event = Event() self.start_event = Event() self.chain.client.inject_stop_event(self.stop_event) self.wal = None self.database_path = config['database_path'] if self.database_path != ':memory:': database_dir = os.path.dirname(config['database_path']) os.makedirs(database_dir, exist_ok=True) self.database_dir = database_dir # Prevent concurrent acces to the same db self.lock_file = os.path.join(self.database_dir, '.lock') self.db_lock = filelock.FileLock(self.lock_file) else: self.database_path = ':memory:' self.database_dir = None self.lock_file = None self.serialization_file = None self.db_lock = None if config['transport_type'] == 'udp': # If the endpoint registration fails the node will quit, this must # finish before starting the transport endpoint_registration_event.join() self.event_poll_lock = gevent.lock.Semaphore() self.start() def start(self): """ Start the node. """ if self.stop_event and self.stop_event.is_set(): self.stop_event.clear() if self.database_dir is not None: self.db_lock.acquire(timeout=0) assert self.db_lock.is_locked # The database may be :memory: storage = sqlite.SQLiteStorage(self.database_path, serialize.PickleSerializer()) self.wal, unapplied_events = wal.restore_from_latest_snapshot( node.state_transition, storage, ) if self.wal.state_manager.current_state is None: block_number = self.chain.block_number() state_change = ActionInitNode( random.Random(), block_number, ) self.wal.log_and_dispatch(state_change, block_number) # On first run Raiden needs to fetch all events for the payment # network, to reconstruct all token network graphs and find opened # channels last_log_block_number = None else: # The `Block` state change is dispatched only after all the events # for that given block have been processed, filters can be safely # installed starting from this position without losing events. last_log_block_number = views.block_number( self.wal.state_manager.current_state) # The time the alarm task is started or the callbacks are installed doesn't # really matter. # # But it is of paramount importance to: # - Install the filters which will be polled by poll_blockchain_events # after the state has been primed, otherwise the state changes won't # have effect. # - Install the filters using the correct from_block value, otherwise # blockchain logs can be lost. self.alarm.register_callback(self._callback_new_block) self.alarm.start() self.install_payment_network_filters( self.default_registry.address, last_log_block_number, ) # Start the transport after the registry is queried to avoid warning # about unknown channels. queueids_to_queues = views.get_all_messagequeues( views.state_from_raiden(self)) self.transport.start(self, queueids_to_queues) # Health check needs the transport layer self.start_neighbours_healthcheck() for event in unapplied_events: on_raiden_event(self, event) self.start_event.set() def start_neighbours_healthcheck(self): for neighbour in views.all_neighbour_nodes( self.wal.state_manager.current_state): if neighbour != ConnectionManager.BOOTSTRAP_ADDR: self.start_health_check_for(neighbour) def stop(self): """ Stop the node. """ # Needs to come before any greenlets joining self.stop_event.set() self.transport.stop_and_wait() self.alarm.stop_async() wait_for = [self.alarm] wait_for.extend(getattr(self.transport, 'greenlets', [])) # We need a timeout to prevent an endless loop from trying to # contact the disconnected client gevent.wait(wait_for, timeout=self.shutdown_timeout) # Filters must be uninstalled after the alarm task has stopped. Since # the events are polled by an alarm task callback, if the filters are # uninstalled before the alarm task is fully stopped the callback # `poll_blockchain_events` will fail. # # We need a timeout to prevent an endless loop from trying to # contact the disconnected client try: with gevent.Timeout(self.shutdown_timeout): self.blockchain_events.uninstall_all_event_listeners() except (gevent.timeout.Timeout, RaidenShuttingDown): pass if self.db_lock is not None: self.db_lock.release() def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, pex(self.address)) def get_block_number(self): return views.block_number(self.wal.state_manager.current_state) def handle_state_change(self, state_change, block_number=None): log.debug('STATE CHANGE', node=pex(self.address), state_change=state_change) if block_number is None: block_number = self.get_block_number() event_list = self.wal.log_and_dispatch(state_change, block_number) for event in event_list: log.debug('EVENT', node=pex(self.address), chain_event=event) on_raiden_event(self, event) return event_list def set_node_network_state(self, node_address, network_state): state_change = ActionChangeNodeNetworkState(node_address, network_state) self.wal.log_and_dispatch(state_change, self.get_block_number()) def start_health_check_for(self, node_address): self.transport.start_health_check(node_address) def _callback_new_block(self, current_block_number): """Called once a new block is detected by the alarm task. Note: This should be called only once per block, otherwise there will be duplicated `Block` state changes in the log. Therefore this method should be called only once a new block is mined with the appropriate block_number argument from the AlarmTask. """ # Raiden relies on blockchain events to update its off-chain state, # therefore some APIs /used/ to forcefully poll for events. # # This was done for APIs which have on-chain side-effects, e.g. # openning a channel, where polling the event is required to update # off-chain state to providing a consistent view to the caller, e.g. # the channel exists after the API call returns. # # That pattern introduced a race, because the events are returned only # once per filter, and this method would be called concurrently by the # API and the AlarmTask. The following lock is necessary, to ensure the # expected side-effects are properly applied (introduced by the commit # 3686b3275ff7c0b669a6d5e2b34109c3bdf1921d) with self.event_poll_lock: for event in self.blockchain_events.poll_blockchain_events(): # These state changes will be procesed with a block_number # which is /larger/ than the NodeState's block_number. on_blockchain_event(self, event, current_block_number) # On restart the Raiden node will re-create the filters with the # ethereum node. These filters will have the from_block set to the # value of the latest Block state change. To avoid missing events # the Block state change is dispatched only after all of the events # have been processed. # # This means on some corner cases a few events may be applied # twice, this will happen if the node crashed and some events have # been processed but the Block state change has not been # dispatched. state_change = Block(current_block_number) self.handle_state_change(state_change, current_block_number) def sign(self, message): """ Sign message inplace. """ if not isinstance(message, SignedMessage): raise ValueError('{} is not signable.'.format(repr(message))) message.sign(self.private_key, self.address) def install_payment_network_filters(self, payment_network_id, from_block=None): proxies = get_relevant_proxies( self.chain, self.address, payment_network_id, ) # Install the filters first to avoid missing changes, as a consequence # some events might be applied twice. self.blockchain_events.add_proxies_listeners(proxies, from_block) token_network_list = list() for manager in proxies.channel_managers: manager_address = manager.address netting_channel_proxies = proxies.channelmanager_nettingchannels[ manager_address] network = get_token_network_state_from_proxies( self, manager, netting_channel_proxies) token_network_list.append(network) payment_network = PaymentNetworkState( payment_network_id, token_network_list, ) state_change = ContractReceiveNewPaymentNetwork(payment_network) self.handle_state_change(state_change) def connection_manager_for_token(self, registry_address, token_address): if not is_binary_address(token_address): raise InvalidAddress('token address is not valid.') known_token_networks = views.get_token_network_addresses_for( self.wal.state_manager.current_state, registry_address, ) if token_address not in known_token_networks: raise InvalidAddress('token is not registered.') manager = self.tokens_to_connectionmanagers.get(token_address) if manager is None: manager = ConnectionManager(self, registry_address, token_address) self.tokens_to_connectionmanagers[token_address] = manager return manager def leave_all_token_networks(self): state_change = ActionLeaveAllNetworks() self.wal.log_and_dispatch(state_change, self.get_block_number()) def close_and_settle(self): log.info('raiden will close and settle all channels now') self.leave_all_token_networks() connection_managers = [ self.tokens_to_connectionmanagers[token_address] for token_address in self.tokens_to_connectionmanagers ] if connection_managers: waiting.wait_for_settle_all_channels( self, self.alarm.wait_time, ) def mediated_transfer_async( self, token_network_identifier, amount, target, identifier, ): """ Transfer `amount` between this node and `target`. This method will start an asyncronous transfer, the transfer might fail or succeed depending on a couple of factors: - Existence of a path that can be used, through the usage of direct or intermediary channels. - Network speed, making the transfer sufficiently fast so it doesn't expire. """ async_result = self.start_mediated_transfer( token_network_identifier, amount, target, identifier, ) return async_result def direct_transfer_async(self, token_network_identifier, amount, target, identifier): """ Do a direct transfer with target. Direct transfers are non cancellable and non expirable, since these transfers are a signed balance proof with the transferred amount incremented. Because the transfer is non cancellable, there is a level of trust with the target. After the message is sent the target is effectively paid and then it is not possible to revert. The async result will be set to False iff there is no direct channel with the target or the payer does not have balance to complete the transfer, otherwise because the transfer is non expirable the async result *will never be set to False* and if the message is sent it will hang until the target node acknowledge the message. This transfer should be used as an optimization, since only two packets are required to complete the transfer (from the payers perspective), whereas the mediated transfer requires 6 messages. """ self.transport.start_health_check(target) if identifier is None: identifier = create_default_identifier() direct_transfer = ActionTransferDirect( token_network_identifier, target, identifier, amount, ) self.handle_state_change(direct_transfer) def start_mediated_transfer( self, token_network_identifier, amount, target, identifier, ): self.transport.start_health_check(target) if identifier is None: identifier = create_default_identifier() assert identifier not in self.identifier_to_results async_result = AsyncResult() self.identifier_to_results[identifier].append(async_result) secret = random_secret() init_initiator_statechange = initiator_init( self, identifier, amount, secret, token_network_identifier, target, ) # TODO: implement the network timeout raiden.config['msg_timeout'] and # cancel the current transfer if it happens (issue #374) # # Dispatch the state change even if there are no routes to create the # wal entry. self.handle_state_change(init_initiator_statechange) return async_result def mediate_mediated_transfer(self, transfer: LockedTransfer): init_mediator_statechange = mediator_init(self, transfer) self.handle_state_change(init_mediator_statechange) def target_mediated_transfer(self, transfer: LockedTransfer): init_target_statechange = target_init(transfer) self.handle_state_change(init_target_statechange)
def __init__(self, name, *args, **kwargs): super(Queue, self).__init__(*args, **kwargs) self.name = name self.__has_content = Event() self.__has_content.clear()
class AceClient(object): def __init__(self, acehost, aceAPIport, aceHTTPport, acehostslist, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current video position self._position = None # Available video position (loaded data) self._position_last = None # Buffered video pieces self._position_buf = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Result for GETCID() self._cidresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Seekback seconds. self._seekback = AceConfig.videoseekback # Did we get START command again? For seekback. self._started_again = False self._idleSince = time.time() self._lock = threading.Condition(threading.Lock()) self._streamReaderConnection = None self._streamReaderState = None self._streamReaderQueue = deque() self._engine_version_code = 0; # Logger logger = logging.getLogger('AceClientimport tracebacknt_init') # Try to connect AceStream engine try: self._socket = telnetlib.Telnet(acehost, aceAPIport, connect_timeout) AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = acehost, aceAPIport, aceHTTPport logger.debug("Successfully connected to AceStream on %s:%d" % (acehost, aceAPIport)) except: for AceEngine in acehostslist: try: self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1], connect_timeout) AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[0], AceEngine[1], AceEngine[2] logger.debug("Successfully connected to AceStream on %s:%d" % (AceEngine[0], AceEngine[1])) break except: logger.debug("The are no alive AceStream on %s:%d" % (AceEngine[0], AceEngine[1])) pass # Spawning recvData greenlet gevent.spawn(self._recvData) gevent.sleep() def __del__(self): # Destructor just calls destroy() method self.destroy() def destroy(self): ''' AceClient Destructor ''' if self._shuttingDown.isSet(): # Already in the middle of destroying return # Logger logger = logging.getLogger("AceClient_destroy") # We should resume video to prevent read greenlet deadlock self._resumeevent.set() # And to prevent getUrl deadlock self._urlresult.set() # Trying to disconnect try: logger.debug("Destroying client...") self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: # Ignore exceptions on destroy pass finally: self._shuttingDown.set() def reset(self): self._started_again = False self._idleSince = time.time() self._streamReaderState = None def _write(self, message): try: logger = logging.getLogger("AceClient_write") logger.debug(message) self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + repr(e)) def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, seekback=0): self._product_key = product_key self._gender = gender self._age = age # Seekback seconds self._seekback = seekback # Logger logger = logging.getLogger("AceClient_aceInit") # Sending HELLO self._write(AceMessage.request.HELLO) if not self._authevent.wait(self._resulttimeout): errmsg = "Authentication timeout. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = "Authentication error. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return logger.debug("aceInit ended") def _getResult(self): # Logger try: result = self._result.get(timeout=self._resulttimeout) if not result: raise AceException("Result not received") except gevent.Timeout: raise AceException("Timeout") return result def START(self, datatype, value): ''' Start video method ''' if self._engine_version_code >= 3010500 and AceConfig.vlcuse: stream_type = 'output_format=hls' + ' transcode_audio=' + str(AceConfig.transcode_audio) \ + ' transcode_mp3=' + str(AceConfig.transcode_mp3) \ + ' transcode_ac3=' + str(AceConfig.transcode_ac3) \ + ' preferred_audio_language=' + AceConfig.preferred_audio_language else: stream_type = 'output_format=http' self._urlresult = AsyncResult() self._write(AceMessage.request.START(datatype.upper(), value, stream_type)) self._getResult() def STOP(self): ''' Stop video method ''' if self._state is not None and self._state != '0': self._result = AsyncResult() self._write(AceMessage.request.STOP) self._getResult() def LOADASYNC(self, datatype, value): self._result = AsyncResult() self._write(AceMessage.request.LOADASYNC(datatype.upper(), random.randint(1, AceConfig.maxconns * 10000), value)) return self._getResult() def GETCONTENTINFO(self, datatype, value): dict = {'torrent': 'url', 'infohash':'infohash', 'raw':'data', 'pid':'content_id'} self.paramsdict={dict[datatype]:value,'developer_id':'0','affiliate_id':'0','zone_id':'0'} contentinfo = self.LOADASYNC(datatype, self.paramsdict) return contentinfo def GETCID(self, datatype, url): contentinfo = self.GETCONTENTINFO(datatype, url) self._cidresult = AsyncResult() self._write(AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0)) cid = self._cidresult.get(True, 5) return '' if not cid or cid == '' else cid[2:] def getUrl(self, timeout=30): # Logger logger = logging.getLogger("AceClient_getURL") try: res = self._urlresult.get(timeout=timeout) return res except gevent.Timeout: errmsg = "Engine response time exceeded. GetURL timeout!" logger.error(errmsg) raise AceException(errmsg) def startStreamReader(self, url, cid, counter, req_headers=None): logger = logging.getLogger("StreamReader") logger.debug("Opening video stream: %s" % url) logger.debug("Get headers from client: %s" % req_headers) self._streamReaderState = 1 try: if url.endswith('.m3u8'): logger.debug("Can't stream HLS in non VLC mode: %s" % url) return None # Sending client hedaers to AceEngine if req_headers.has_key('range'): del req_headers['range'] connection = self._streamReaderConnection = requests.get(url, headers=req_headers, stream=True) if connection.status_code not in (200, 206): logger.error("Failed to open video stream %s" % url) return None with self._lock: self._streamReaderState = 2 self._lock.notifyAll() while True: data = None clients = counter.getClients(cid) try: data = connection.raw.read(AceConfig.readchunksize) except: break; if data and clients: with self._lock: if len(self._streamReaderQueue) == AceConfig.readchunksize: self._streamReaderQueue.popleft() self._streamReaderQueue.append(data) for c in clients: try: c.addChunk(data, 5.0) except Queue.Full: if len(clients) > 1: logger.debug("Disconnecting client: %s" % str(c)) c.destroy() elif not clients: logger.debug("All clients disconnected - closing video stream") break else: logger.warning("No data received") break except requests.exceptions.ConnectionError: logger.error("Failed to open video stream") logger.error(traceback.format_exc()) except: logger.error(traceback.format_exc()) if counter.getClients(cid): logger.error("Failed to read video stream") finally: self.closeStreamReader() with self._lock: self._streamReaderState = 3 self._lock.notifyAll() counter.deleteAll(cid) def closeStreamReader(self): logger = logging.getLogger("StreamReader") c = self._streamReaderConnection if c: self._streamReaderConnection = None c.close() logger.debug("Video stream closed") self._streamReaderQueue.clear() def getPlayEvent(self, timeout=None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' return self._resumeevent.wait(timeout=timeout) def pause(self): self._write(AceMessage.request.PAUSE) def play(self): self._write(AceMessage.request.PLAY) def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() try: self._recvbuffer = self._socket.read_until("\r\n") self._recvbuffer = self._recvbuffer.strip() logger.debug('<<< ' + self._recvbuffer) except: # If something happened during read, abandon reader. if not self._shuttingDown.isSet(): logger.error("Exception at socket read") self._shuttingDown.set() return if self._recvbuffer: # Parsing everything only if the string is not empty if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'version_code=' in self._recvbuffer: v = self._recvbuffer.find('version_code=') self._engine_version_code = int(self._recvbuffer[v+13:v+20]) if 'key=' in self._recvbuffer: self._request_key_begin = self._recvbuffer.find('key=') self._request_key = \ self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14] try: self._write(AceMessage.request.READY_key( self._request_key, self._product_key)) except requests.exceptions.ConnectionError as e: logger.error("Can't connect to keygen server! " + \ repr(e)) self._auth = False self._authevent.set() self._request_key = None else: self._write(AceMessage.request.READY_nokey) elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): # NOTREADY logger.error("Ace is not ready. Wrong auth?") self._auth = False self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.LOADRESP): # LOADRESP _contentinfo_raw = self._recvbuffer.split()[2:] _contentinfo_raw = ' '.join(_contentinfo_raw) _contentinfo = json.loads(_contentinfo_raw) if _contentinfo.get('status') == 100: logger.error("LOADASYNC returned error with message: %s" % _contentinfo.get('message')) self._result.set(False) else: logger.debug("Content info: %s", _contentinfo) self._result.set(_contentinfo) elif self._recvbuffer.startswith(AceMessage.response.START): # START if not self._seekback or self._started_again or not self._recvbuffer.endswith(' stream=1'): # If seekback is disabled, we use link in first START command. # If seekback is enabled, we wait for first START command and # ignore it, then do seekback in first EVENT position command # AceStream sends us STOP and START again with new link. # We use only second link then. try: self._url = self._recvbuffer.split()[1] self._urlresult.set(self._url) self._resumeevent.set() except IndexError as e: self._url = None else: logger.debug("START received. Waiting for %s." % AceMessage.response.LIVEPOS) # STOP elif self._recvbuffer.startswith(AceMessage.response.STOP): pass # SHUTDOWN elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): logger.debug("Got SHUTDOWN from engine") self._socket.close() return # AUTH elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write( AceMessage.request.USERDATA(self._gender, self._age)) except: pass self._authevent.set() # GETUSERDATA elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") # LIVEPOS elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS): self._position = self._recvbuffer.split() self._position_last = self._position[2].split('=')[1] self._position_buf = self._position[9].split('=')[1] self._position = self._position[4].split('=')[1] if self._seekback and not self._started_again: self._write(AceMessage.request.LIVESEEK(str(int(self._position_last) - self._seekback))) logger.debug('Seeking back') self._started_again = True # DOWNLOADSTOP elif self._recvbuffer.startswith(AceMessage.response.DOWNLOADSTOP): self._state = self._recvbuffer.split()[1] # STATE elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] # STATUS elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split(';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to " + self._status) if self._status == 'main:err': logger.error( self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) self._urlresult.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) elif self._status == 'main:starting': self._result.set(True) elif self._status == 'main:idle': self._result.set(True) # PAUSE elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() # RESUME elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") gevent.sleep() # PAUSE/RESUME delay self._resumeevent.set() # CID elif self._recvbuffer.startswith('##') or len(self._recvbuffer) == 0: self._cidresult.set(self._recvbuffer) logger.debug("CID: %s" %self._recvbuffer)
def __init__( self, chain: BlockChainService, default_registry: Registry, default_secret_registry: SecretRegistry, private_key_bin, transport, config, discovery=None, ): if not isinstance(private_key_bin, bytes) or len(private_key_bin) != 32: raise ValueError('invalid private_key') invalid_timeout = ( config['settle_timeout'] < NETTINGCHANNEL_SETTLE_TIMEOUT_MIN or config['settle_timeout'] > NETTINGCHANNEL_SETTLE_TIMEOUT_MAX) if invalid_timeout: raise ValueError('settle_timeout must be in range [{}, {}]'.format( NETTINGCHANNEL_SETTLE_TIMEOUT_MIN, NETTINGCHANNEL_SETTLE_TIMEOUT_MAX, )) self.tokens_to_connectionmanagers = dict() self.identifier_to_results = defaultdict(list) self.chain: BlockChainService = chain self.default_registry = default_registry self.default_secret_registry = default_secret_registry self.config = config self.privkey = private_key_bin self.address = privatekey_to_address(private_key_bin) if config['transport_type'] == 'udp': endpoint_registration_event = gevent.spawn( discovery.register, self.address, config['external_ip'], config['external_port'], ) endpoint_registration_event.link_exception( endpoint_registry_exception_handler) self.private_key = PrivateKey(private_key_bin) self.pubkey = self.private_key.public_key.format(compressed=False) self.transport = transport self.blockchain_events = BlockchainEvents() self.alarm = AlarmTask(chain) self.shutdown_timeout = config['shutdown_timeout'] self.stop_event = Event() self.start_event = Event() self.chain.client.inject_stop_event(self.stop_event) self.wal = None self.database_path = config['database_path'] if self.database_path != ':memory:': database_dir = os.path.dirname(config['database_path']) os.makedirs(database_dir, exist_ok=True) self.database_dir = database_dir # Prevent concurrent acces to the same db self.lock_file = os.path.join(self.database_dir, '.lock') self.db_lock = filelock.FileLock(self.lock_file) else: self.database_path = ':memory:' self.database_dir = None self.lock_file = None self.serialization_file = None self.db_lock = None if config['transport_type'] == 'udp': # If the endpoint registration fails the node will quit, this must # finish before starting the transport endpoint_registration_event.join() self.event_poll_lock = gevent.lock.Semaphore() self.start()
def get(self, key): event = Event() item = {"operation": "get", "key": key, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event)
def __init__(self, acehost, aceAPIport, aceHTTPport, acehostslist, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current video position self._position = None # Available video position (loaded data) self._position_last = None # Buffered video pieces self._position_buf = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Result for GETCID() self._cidresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Seekback seconds. self._seekback = AceConfig.videoseekback # Did we get START command again? For seekback. self._started_again = False self._idleSince = time.time() self._lock = threading.Condition(threading.Lock()) self._streamReaderConnection = None self._streamReaderState = None self._streamReaderQueue = deque() self._engine_version_code = 0; # Logger logger = logging.getLogger('AceClientimport tracebacknt_init') # Try to connect AceStream engine try: self._socket = telnetlib.Telnet(acehost, aceAPIport, connect_timeout) AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = acehost, aceAPIport, aceHTTPport logger.debug("Successfully connected to AceStream on %s:%d" % (acehost, aceAPIport)) except: for AceEngine in acehostslist: try: self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1], connect_timeout) AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[0], AceEngine[1], AceEngine[2] logger.debug("Successfully connected to AceStream on %s:%d" % (AceEngine[0], AceEngine[1])) break except: logger.debug("The are no alive AceStream on %s:%d" % (AceEngine[0], AceEngine[1])) pass # Spawning recvData greenlet gevent.spawn(self._recvData) gevent.sleep()
class MeekSession(RelaySession): conn_pool = HTTPClientPool() def __init__(self, socksconn, meek, timeout): super(MeekSession, self).__init__(socksconn) self.sessionid = session_id() self.meek = meek self.meektimeout = timeout self.relay = self.meek.select_relay() self.ca_certs = self.meek.ca_certs self.httpclient = self.conn_pool.get(self.relay, self.ca_certs, self.meektimeout) self.udpsock = None self.allsocks = [self.socksconn] self.l2m_queue = Queue() self.m2l_queue = Queue() self.m_notifier = Event() self.l_notifier = Event() self.finish = Event() self.m_notifier.clear() self.l_notifier.clear() self.finish.clear() self.timer = SharedTimer(self.meektimeout) def _stream_response(self, response): try: chunk = response.read(MAX_PAYLOAD_LENGTH) while chunk: log.debug("%s streaming DOWN %d bytes" % (self.sessionid, len(chunk))) yield chunk, "" chunk = response.read(MAX_PAYLOAD_LENGTH) except GeneratorExit: response.release() raise StopIteration def meek_response(self, response, stream): if stream: return self._stream_response(response) data = response.read() response.release() if not data: return [("", "")] if not self.udpsock: return [(data, "")] # parse UDP packets log.debug("%s DOWN %d bytes" % (self.sessionid, len(data))) lengths = get_meek_meta(response.headers, HEADER_UDP_PKTS).split(",") pos = 0 pkts = [] for length in lengths: nxt = pos + int(length) pkts.append((data[pos:nxt], "")) pos = nxt return pkts def meek_roundtrip(self, pkts): headers = { HEADER_SESSION_ID: self.sessionid, HEADER_MSGTYPE: MSGTYPE_DATA, 'Host': self.relay.hostname, 'Content-Type': "application/octet-stream", 'Connection': "Keep-Alive", } stream = False if not self.udpsock and "stream" in self.relay.properties: stream = True headers[HEADER_MODE] = MODE_STREAM if pkts and self.udpsock: lengths = str(",".join([str(len(p)) for p in pkts])) headers[HEADER_UDP_PKTS] = lengths data = "".join(pkts) headers['Content-Length'] = str(len(data)) for _ in range(CLIENT_MAX_TRIES): try: log.debug("%s UP %d bytes" % (self.sessionid, len(data))) resp = self.httpclient.post("/", body=data, headers=headers) if resp.status_code != 200: # meek server always give 200, so all non-200s mean external issues. continue err = get_meek_meta(resp.headers, HEADER_ERROR) if err: return [("", err)] else: try: return self.meek_response(resp, stream) except Exception as ex: log.error( "[Exception][meek_roundtrip - meek_response]: %s" % str(ex)) resp.release() return [("", "Data Format Error")] except socket.timeout: # @UndefinedVariable return [("", "timeout")] except Exception as ex: log.error("[Exception][meek_roundtrip]: %s" % str(ex)) gevent.sleep(CLIENT_RETRY_DELAY) self.relay.failure += 1 return [("", "Max Retry (%d) Exceeded" % CLIENT_MAX_TRIES)] def meek_sendrecv(self): pkts = [] datalen = 0 while not self.l2m_queue.empty(): pkt = self.l2m_queue.get() pkts.append(pkt) datalen += len(pkt) if datalen >= MAX_PAYLOAD_LENGTH: for (resp, err) in self.meek_roundtrip(pkts): yield (resp, err) if err or not resp: return pkts = [] datalen = 0 for (resp, err) in self.meek_roundtrip(pkts): yield (resp, err) if err or not resp: return def meek_relay(self): for (resp, err) in self.meek_sendrecv(): if err: return err if resp: self.m2l_queue.put(resp) self.l_notifier.set() return "" def meek_relay_thread(self): interval = CLIENT_INITIAL_POLL_INTERVAL while not self.finish.is_set(): try: hasdata = self.m_notifier.wait(timeout=interval) self.m_notifier.clear() err = self.meek_relay() if err: break if not hasdata: interval *= CLIENT_POLL_INTERVAL_MULTIPLIER if interval > CLIENT_MAX_POLL_INTERVAL: interval = CLIENT_MAX_POLL_INTERVAL except Exception as ex: log.error("[Exception][meek_relay_thread]: %s" % str(ex)) break self.finish.set() def write_to_client(self, data): if self.udpsock: self.udpsock.sendto(data, self.last_clientaddr) else: self.socksconn.sendall(data) def meek_write_to_client_thread(self): while not self.finish.is_set(): try: hasdata = self.l_notifier.wait( timeout=CLIENT_MAX_POLL_INTERVAL) self.l_notifier.clear() if not hasdata: self.timer.count(CLIENT_MAX_POLL_INTERVAL) if self.timer.timeout(): break else: self.timer.reset() while not self.m2l_queue.empty(): data = self.m2l_queue.get() if data: self.write_to_client(data) except Exception as ex: log.error("[Exception][meek_write_to_client_thread]: %s" % str(ex)) break self.finish.set() def read_from_client(self, timeout): readable, _, _ = select.select(self.allsocks, [], [], CLIENT_MAX_POLL_INTERVAL) if not readable: return None if self.socksconn in readable: if self.udpsock: raise RelaySessionError( "unexcepted read-event from tcp socket in UDP session") data = self.socksconn.recv(MAX_PAYLOAD_LENGTH) if not data: raise RelaySessionError("peer closed") return data if self.udpsock and self.udpsock in readable: data, addr = self.udpsock.recvfrom(MAX_PAYLOAD_LENGTH) if not self.valid_udp_client(addr): return None else: self.last_clientaddr = addr return data def meek_read_from_client_thread(self): while not self.finish.is_set(): try: data = self.read_from_client(CLIENT_MAX_POLL_INTERVAL) if not data: self.timer.count(CLIENT_MAX_POLL_INTERVAL) if self.timer.timeout(): break else: self.timer.reset() self.l2m_queue.put(data) self.m_notifier.set() except Exception as ex: log.error("[Exception][meek_read_from_client_thread]: %s" % str(ex)) break self.finish.set() def proc_tcp_request(self, req): self.l2m_queue.put(req.pack()) def relay_tcp(self): read_thread = gevent.spawn(self.meek_read_from_client_thread) write_thread = gevent.spawn(self.meek_write_to_client_thread) relay_thread = gevent.spawn(self.meek_relay_thread) # notify relay to send request self.m_notifier.set() [t.join() for t in (read_thread, write_thread, relay_thread)] log.info("Session %s Ended" % self.sessionid) def valid_udp_client(self, addr): if self.client_associate[0] == "0.0.0.0" or \ self.client_associate[0] == "::": return True if self.client_associate == addr: return True return False def cmd_udp_associate(self, req): self.client_associate = (req.dstaddr, req.dstport) self.last_clientaddr = self.client_associate for (resp, err) in self.meek_roundtrip([req.pack()]): if err: return if resp: Reply(resp) self.udpsock = bind_local_udp(self.socksconn) if not self.udpsock: request_fail(self.socksconn, req, GENERAL_SOCKS_SERVER_FAILURE) return self.track_sock(self.udpsock) read_thread = gevent.spawn(self.meek_read_from_client_thread) write_thread = gevent.spawn(self.meek_write_to_client_thread) relay_thread = gevent.spawn(self.meek_relay_thread) request_success(self.socksconn, *sock_addr_info(self.udpsock)) [t.join() for t in (read_thread, write_thread, relay_thread)] log.info("Session %s Ended" % self.sessionid) def meek_terminate(self): headers = { HEADER_SESSION_ID: self.sessionid, HEADER_MSGTYPE: MSGTYPE_TERMINATE, #'Content-Type': "application/octet-stream", 'Content-Length': "0", 'Connection': "Keep-Alive", 'Host': self.relay.hostname, } try: self.httpclient.post("/", data="", headers=headers) except: pass def clean(self): self.meek_terminate() for sock in self.allsocks: sock.close() #self.httpclient.close() self.conn_pool.release(self.relay, self.httpclient)
def _run_app(self): from raiden.ui.console import Console from raiden.api.python import RaidenAPI # this catches exceptions raised when waiting for the stalecheck to complete try: app_ = run_app(**self._options) except (EthNodeCommunicationError, RequestsConnectionError): print(ETHEREUM_NODE_COMMUNICATION_ERROR) sys.exit(1) domain_list = [] if self._options['rpccorsdomain']: if ',' in self._options['rpccorsdomain']: for domain in self._options['rpccorsdomain'].split(','): domain_list.append(str(domain)) else: domain_list.append(str(self._options['rpccorsdomain'])) self._raiden_api = RaidenAPI(app_.raiden) api_server = None if self._options['rpc']: rest_api = RestAPI(self._raiden_api) api_server = APIServer( rest_api, cors_domain_list=domain_list, web_ui=self._options['web_ui'], eth_rpc_endpoint=self._options['eth_rpc_endpoint'], ) (api_host, api_port) = split_endpoint(self._options['api_address']) try: api_server.start(api_host, api_port) except APIServerPortInUseError: print( 'ERROR: API Address %s:%s is in use. ' 'Use --api-address <host:port> to specify port to listen on.' % (api_host, api_port), ) sys.exit(1) print( 'The Raiden API RPC server is now running at http://{}:{}/.\n\n' 'See the Raiden documentation for all available endpoints at\n' 'http://raiden-network.readthedocs.io/en/stable/rest_api.html'. format( api_host, api_port, ), ) if self._options['console']: console = Console(app_) console.start() # spawn a greenlet to handle the version checking gevent.spawn(check_version) # spawn a greenlet to handle the gas reserve check gevent.spawn(check_gas_reserve, app_.raiden) self._startup_hook() # wait for interrupt event = Event() gevent.signal(signal.SIGQUIT, event.set) gevent.signal(signal.SIGTERM, event.set) gevent.signal(signal.SIGINT, event.set) try: event.wait() print('Signal received. Shutting down ...') except (EthNodeCommunicationError, RequestsConnectionError): print(ETHEREUM_NODE_COMMUNICATION_ERROR) sys.exit(1) except RaidenError as ex: click.secho(f'FATAL: {ex}', fg='red') except Exception as ex: with NamedTemporaryFile( 'w', prefix= f'raiden-exception-{datetime.utcnow():%Y-%m-%dT%H-%M}', suffix='.txt', delete=False, ) as traceback_file: traceback.print_exc(file=traceback_file) click.secho( f'FATAL: An unexpected exception occured. ' f'A traceback has been written to {traceback_file.name}\n' f'{ex}', fg='red', ) finally: self._shutdown_hook() if api_server: api_server.stop() return app_
class MDSThrasher(Greenlet, Thrasher, object): """ MDSThrasher:: The MDSThrasher thrashes MDSs during execution of other tasks (workunits, etc). The config is optional. Many of the config parameters are a a maximum value to use when selecting a random value from a range. To always use the maximum value, set no_random to true. The config is a dict containing some or all of: max_thrash: [default: 1] the maximum number of active MDSs per FS that will be thrashed at any given time. max_thrash_delay: [default: 30] maximum number of seconds to delay before thrashing again. max_replay_thrash_delay: [default: 4] maximum number of seconds to delay while in the replay state before thrashing. max_revive_delay: [default: 10] maximum number of seconds to delay before bringing back a thrashed MDS. randomize: [default: true] enables randomization and use the max/min values seed: [no default] seed the random number generator thrash_in_replay: [default: 0.0] likelihood that the MDS will be thrashed during replay. Value should be between 0.0 and 1.0. thrash_max_mds: [default: 0.05] likelihood that the max_mds of the mds cluster will be modified to a value [1, current) or (current, starting max_mds]. Value should be between 0.0 and 1.0. thrash_while_stopping: [default: false] thrash an MDS while there are MDS in up:stopping (because max_mds was changed and some MDS were deactivated). thrash_weights: allows specific MDSs to be thrashed more/less frequently. This option overrides anything specified by max_thrash. This option is a dict containing mds.x: weight pairs. For example, [mds.a: 0.7, mds.b: 0.3, mds.c: 0.0]. Each weight is a value from 0.0 to 1.0. Any MDSs not specified will be automatically given a weight of 0.0 (not thrashed). For a given MDS, by default the trasher delays for up to max_thrash_delay, trashes, waits for the MDS to recover, and iterates. If a non-zero weight is specified for an MDS, for each iteration the thrasher chooses whether to thrash during that iteration based on a random value [0-1] not exceeding the weight of that MDS. Examples:: The following example sets the likelihood that mds.a will be thrashed to 80%, mds.b to 20%, and other MDSs will not be thrashed. It also sets the likelihood that an MDS will be thrashed in replay to 40%. Thrash weights do not have to sum to 1. tasks: - ceph: - mds_thrash: thrash_weights: - mds.a: 0.8 - mds.b: 0.2 thrash_in_replay: 0.4 - ceph-fuse: - workunit: clients: all: [suites/fsx.sh] The following example disables randomization, and uses the max delay values: tasks: - ceph: - mds_thrash: max_thrash_delay: 10 max_revive_delay: 1 max_replay_thrash_delay: 4 """ def __init__(self, ctx, manager, config, fs, max_mds): super(MDSThrasher, self).__init__() self.config = config self.ctx = ctx self.logger = log.getChild('fs.[{f}]'.format(f=fs.name)) self.fs = fs self.manager = manager self.max_mds = max_mds self.name = 'thrasher.fs.[{f}]'.format(f=fs.name) self.stopping = Event() self.randomize = bool(self.config.get('randomize', True)) self.thrash_max_mds = float(self.config.get('thrash_max_mds', 0.05)) self.max_thrash = int(self.config.get('max_thrash', 1)) self.max_thrash_delay = float(self.config.get('thrash_delay', 120.0)) self.thrash_in_replay = float( self.config.get('thrash_in_replay', False)) assert self.thrash_in_replay >= 0.0 and self.thrash_in_replay <= 1.0, 'thrash_in_replay ({v}) must be between [0.0, 1.0]'.format( v=self.thrash_in_replay) self.max_replay_thrash_delay = float( self.config.get('max_replay_thrash_delay', 4.0)) self.max_revive_delay = float(self.config.get('max_revive_delay', 10.0)) def _run(self): try: self.do_thrash() except Exception as e: # Log exceptions here so we get the full backtrace (gevent loses them). # Also allow successful completion as gevent exception handling is a broken mess: # # 2017-02-03T14:34:01.259 CRITICAL:root: File "gevent.libev.corecext.pyx", line 367, in gevent.libev.corecext.loop.handle_error (src/gevent/libev/gevent.corecext.c:5051) # File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/gevent/hub.py", line 558, in handle_error # self.print_exception(context, type, value, tb) # File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/gevent/hub.py", line 605, in print_exception # traceback.print_exception(type, value, tb, file=errstream) # File "/usr/lib/python2.7/traceback.py", line 124, in print_exception # _print(file, 'Traceback (most recent call last):') # File "/usr/lib/python2.7/traceback.py", line 13, in _print # file.write(str+terminator) # 2017-02-03T14:34:01.261 CRITICAL:root:IOError self.exception = e self.logger.exception("exception:") # allow successful completion so gevent doesn't see an exception... def log(self, x): """Write data to the logger assigned to MDSThrasher""" self.logger.info(x) def stop(self): self.stopping.set() def kill_mds(self, mds): if self.config.get('powercycle'): (remote, ) = (self.ctx.cluster.only( 'mds.{m}'.format(m=mds)).remotes.iterkeys()) self.log('kill_mds on mds.{m} doing powercycle of {s}'.format( m=mds, s=remote.name)) self._assert_ipmi(remote) remote.console.power_off() else: self.ctx.daemons.get_daemon('mds', mds).stop() @staticmethod def _assert_ipmi(remote): assert remote.console.has_ipmi_credentials, ( "powercycling requested but RemoteConsole is not " "initialized. Check ipmi config.") def revive_mds(self, mds): """ Revive mds -- do an ipmpi powercycle (if indicated by the config) and then restart. """ if self.config.get('powercycle'): (remote, ) = (self.ctx.cluster.only( 'mds.{m}'.format(m=mds)).remotes.iterkeys()) self.log('revive_mds on mds.{m} doing powercycle of {s}'.format( m=mds, s=remote.name)) self._assert_ipmi(remote) remote.console.power_on() self.manager.make_admin_daemon_dir(self.ctx, remote) args = [] self.ctx.daemons.get_daemon('mds', mds).restart(*args) def wait_for_stable(self, rank=None, gid=None): self.log('waiting for mds cluster to stabilize...') for itercount in itertools.count(): status = self.fs.status() max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds'] ranks = list(status.get_ranks(self.fs.id)) stopping = filter(lambda info: "up:stopping" == info['state'], ranks) actives = filter( lambda info: "up:active" == info['state'] and "laggy_since" not in info, ranks) if not bool(self.config.get('thrash_while_stopping', False)) and len(stopping) > 0: if itercount % 5 == 0: self.log( 'cluster is considered unstable while MDS are in up:stopping (!thrash_while_stopping)' ) else: if rank is not None: try: info = status.get_rank(self.fs.id, rank) if info['gid'] != gid and "up:active" == info['state']: self.log( 'mds.{name} has gained rank={rank}, replacing gid={gid}' .format(name=info['name'], rank=rank, gid=gid)) return status except: pass # no rank present if len(actives) >= max_mds: # no replacement can occur! self.log( "cluster has %d actives (max_mds is %d), no MDS can replace rank %d" .format(len(actives), max_mds, rank)) return status else: if len(actives) == max_mds: self.log( 'mds cluster has {count} alive and active, now stable!' .format(count=len(actives))) return status, None if itercount > 300 / 2: # 5 minutes raise RuntimeError('timeout waiting for cluster to stabilize') elif itercount % 5 == 0: self.log('mds map: {status}'.format(status=status)) else: self.log('no change') sleep(2) def do_thrash(self): """ Perform the random thrashing action """ self.log('starting mds_do_thrash for fs {fs}'.format(fs=self.fs.name)) stats = { "max_mds": 0, "deactivate": 0, "kill": 0, } while not self.stopping.is_set(): delay = self.max_thrash_delay if self.randomize: delay = random.randrange(0.0, self.max_thrash_delay) if delay > 0.0: self.log('waiting for {delay} secs before thrashing'.format( delay=delay)) self.stopping.wait(delay) if self.stopping.is_set(): continue status = self.fs.status() if random.random() <= self.thrash_max_mds: max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds'] options = range(1, max_mds) + range(max_mds + 1, self.max_mds + 1) if len(options) > 0: sample = random.sample(options, 1) new_max_mds = sample[0] self.log('thrashing max_mds: %d -> %d' % (max_mds, new_max_mds)) self.fs.set_max_mds(new_max_mds) stats['max_mds'] += 1 self.wait_for_stable() count = 0 for info in status.get_ranks(self.fs.id): name = info['name'] label = 'mds.' + name rank = info['rank'] gid = info['gid'] # if thrash_weights isn't specified and we've reached max_thrash, # we're done count = count + 1 if 'thrash_weights' not in self.config and count > self.max_thrash: break weight = 1.0 if 'thrash_weights' in self.config: weight = self.config['thrash_weights'].get(label, '0.0') skip = random.randrange(0.0, 1.0) if weight <= skip: self.log( 'skipping thrash iteration with skip ({skip}) > weight ({weight})' .format(skip=skip, weight=weight)) continue self.log('kill {label} (rank={rank})'.format(label=label, rank=rank)) self.kill_mds(name) stats['kill'] += 1 # wait for mon to report killed mds as crashed last_laggy_since = None itercount = 0 while True: status = self.fs.status() info = status.get_mds(name) if not info: break if 'laggy_since' in info: last_laggy_since = info['laggy_since'] break if any([(f == name) for f in status.get_fsmap(self.fs.id) ['mdsmap']['failed']]): break self.log( 'waiting till mds map indicates {label} is laggy/crashed, in failed state, or {label} is removed from mdsmap' .format(label=label)) itercount = itercount + 1 if itercount > 10: self.log('mds map: {status}'.format(status=status)) sleep(2) if last_laggy_since: self.log( '{label} reported laggy/crashed since: {since}'.format( label=label, since=last_laggy_since)) else: self.log('{label} down, removed from mdsmap'.format( label=label, since=last_laggy_since)) # wait for a standby mds to takeover and become active status = self.wait_for_stable(rank, gid) # wait for a while before restarting old active to become new # standby delay = self.max_revive_delay if self.randomize: delay = random.randrange(0.0, self.max_revive_delay) self.log( 'waiting for {delay} secs before reviving {label}'.format( delay=delay, label=label)) sleep(delay) self.log('reviving {label}'.format(label=label)) self.revive_mds(name) for itercount in itertools.count(): if itercount > 300 / 2: # 5 minutes raise RuntimeError('timeout waiting for MDS to revive') status = self.fs.status() info = status.get_mds(name) if info and info['state'] in ('up:standby', 'up:standby-replay', 'up:active'): self.log('{label} reported in {state} state'.format( label=label, state=info['state'])) break self.log( 'waiting till mds map indicates {label} is in active, standby or standby-replay' .format(label=label)) sleep(2) for stat in stats: self.log("stat['{key}'] = {value}".format(key=stat, value=stats[stat]))
class CommandDispatcher(object): #this class contains a queue where request def __init__(self, vbaware, verbose=False): #have a queue , in case of not my vbucket error #let's reinitialize the config/memcached socket connections ? self.queue = Queue(10000) self.status = "initialized" self.vbaware = vbaware self.reconfig_callback = self.vbaware.reconfig_vbucket_map self.start_connection_callback = self.vbaware.start_vbucket_connection self.restart_connection_callback = self.vbaware.restart_vbucket_connection self.verbose = verbose self.log = logger.logger("CommandDispatcher") self._dispatcher_stopped_event = Event() def put(self, item): try: self.queue.put(item, False) except Full: #TODO: add a better error message here raise Exception("queue is full") def shutdown(self): if self.status != "shutdown": self.status = "shutdown" if self.verbose: self.log.info("dispatcher shutdown command received") self._dispatcher_stopped_event.wait(2) def reconfig_completed(self): self.status = "ok" def dispatch(self): while self.status != "shutdown" or (self.status == "shutdown" and self.queue.qsize() > 0): #wait if its reconfiguring the vbucket-map if self.status == "vbucketmap-configuration": continue try: item = self.queue.get(block=True, timeout=1) if item: try: self.do(item) # do will only raise not_my_vbucket_exception, # EOF and socket.error except MemcachedError as ex: # if we get a not_my_vbucket then requeue item # with fast forward map vbucket self.log.error(ex) self.reconfig_callback(ex.vbucket) self.start_connection_callback(ex.vbucket) item["fastforward"] = True self.queue.put(item) except exceptions.EOFError as ex: # we go an EOF error, restart the connection self.log.error(ex) self.restart_connection_callback(ex.vbucket) self.queue.put(item) except socket.error as ex: # we got a socket error, restart the connection self.log.error(ex) self.restart_connection_callback(ex.vbucket) self.queue.put(item) except Empty: pass if self.verbose: self.log.info("dispatcher stopped") self._dispatcher_stopped_event.set() def _raise_if_recoverable(self, ex, item): if isinstance(ex, MemcachedError) and ex.status == 7: ex.vbucket = item["vbucket"] print ex self.log.error( "got not my vb error. key: {0}, vbucket: {1}".format( item["key"], item["vbucket"])) raise ex if isinstance(ex, exceptions.EOFError): ex.vbucket = item["vbucket"] print ex self.log.error("got EOF") raise ex if isinstance(ex, socket.error): ex.vbucket = item["vbucket"] print ex self.log.error("got socket.error") raise ex item["response"]["error"] = ex def do(self, item): #find which vbucket this belongs to and then run the operation on that ? if "key" in item: item["vbucket"] = self.vbaware.vbucketid(item["key"]) if not "fastforward" in item: item["fastforward"] = False item["response"]["return"] = None if item["operation"] == "get": key = item["key"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).get(key) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "set": key = item["key"] expiry = item["expiry"] flags = item["flags"] value = item["value"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).set(key, expiry, flags, value) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "add": key = item["key"] expiry = item["expiry"] flags = item["flags"] value = item["value"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).add(key, expiry, flags, value) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "replace": key = item["key"] expiry = item["expiry"] flags = item["flags"] value = item["value"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).replace(key, expiry, flags, value) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "delete": key = item["key"] cas = item["cas"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).delete(key, cas) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "prepend": key = item["key"] cas = item["cas"] value = item["value"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).prepend(key, value, cas) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "append": key = item["key"] cas = item["cas"] value = item["value"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).append(key, value, cas) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "getl": key = item["key"] expiry = item["expiry"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).getl(key, expiry) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "gat": key = item["key"] expiry = item["expiry"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).gat(key, expiry) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "touch": key = item["key"] expiry = item["expiry"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).touch(key, expiry) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "incr": key = item["key"] amount = item["amount"] init = item["init"] expiry = item["expiry"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).incr(key, amount, init, expiry) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "decr": key = item["key"] amount = item["amount"] init = item["init"] expiry = item["expiry"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).decr(key, amount, init, expiry) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set() elif item["operation"] == "cas": key = item["key"] expiry = item["expiry"] flags = item["flags"] old_value = item["old_value"] value = item["value"] try: item["response"]["return"] = self.vbaware.memcached( key, item["fastforward"]).cas(key, expiry, flags, old_value, value) except Exception as ex: self._raise_if_recoverable(ex, item) item["event"].set()
def __init__(self, web3, mine_sleep=1): super().__init__() self.web3 = web3 self.mine_sleep = mine_sleep self.stop = Event()
# pylint: disable=E0401 from shutil import copyfile from gevent.event import Event import sys sys.path.append('./python') from config import config config.videoPreview = '' config.progressDetail = -1 from worker import context from video import SR_vid context.stopFlag = Event() context.shared = None video = 'upload\\realshort.mp4' copyfile('test\\realshort.mp4', video) steps = [{'op': 'decode'}, {'op': 'range'}, {'codec': 'h264_nvenc -pix_fmt yuv420p', 'op': 'encode', 'file': 'download/realshort.mkv'}] print(SR_vid(video, False, *steps))