class ServerRack(object):
def __init__(self, servers):
self.servers = servers
self.ev = Event()
def start(self):
started = []
try:
for server in self.servers[:]:
server.start()
started.append(server)
name = getattr(server, 'name', None) or server.__class__.__name__ or 'Server'
except:
self.stop(started)
raise
def serve_forever(self):
self.start()
self.ev.wait()
def stop(self, servers=None):
if servers is None:
servers = self.servers[:]
for server in servers:
try:
server.stop()
except:
if hasattr(server, 'loop'): #gevent >= 1.0
server.loop.handle_error(server.stop, *sys.exc_info())
else: # gevent <= 0.13
import traceback
traceback.print_exc()
self.ev.set()
class ConditionPoller(Thread):
"""
generic polling mechanism: every interval seconds, check if condition returns a true value. if so, pass the value to callback
if condition or callback raise exception, stop polling.
"""
def __init__(self, condition, condition_callback, exception_callback, interval):
self.polling_interval = interval
self._shutdown_now = Event()
self._condition = condition
self._callback = condition_callback
self._on_exception = exception_callback
super(ConditionPoller,self).__init__()
def shutdown(self):
self.is_shutting_down = True
self._shutdown_now.set()
def run(self):
try:
while not self._shutdown_now.is_set():
self._check_condition()
self._shutdown_now.wait(self.polling_interval)
except:
log.error('thread failed', exc_info=True)
def _check_condition(self):
try:
value = self._condition()
if value:
self._callback(value)
except Exception as e:
log.debug('stopping poller after exception', exc_info=True)
self.shutdown()
if self._on_exception:
self._on_exception(e)
def start(self):
super(ConditionPoller,self).start()
def events():
try:
last = float(request.get_cookie('interconnect_last_event'))
except Exception:
last = time.time()
evt = Event()
events_history[0][1] > last and evt.set()
event_waiters.add(evt)
success = evt.wait(timeout=30)
event_waiters.discard(evt)
response.set_header('Content-Type', 'application/json')
response.set_header('Pragma', 'no-cache')
response.set_header('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate')
response.set_header('Expires', 'Thu, 01 Dec 1994 16:00:00 GMT')
success and response.set_cookie('interconnect_last_event', '%.5f' % time.time())
data = []
for e in events_history:
if e[1] > last:
data.append(e[0])
else:
break
data = list(reversed(data))
return json.dumps(data)
class BlackBerryPushService(object):
def __init__(self, app_id, password, push_url):
self.app_id = app_id
self.password = password
self.push_url = push_url
self._send_queue = Queue()
self._send_queue_cleared = Event()
self.log = logging.getLogger('pulsus.service.bbp')
def _send_loop(self):
self._send_greenlet = gevent.getcurrent()
try:
self.log.info("BlackBerry Push service started")
while True:
notification = self._send_queue.get()
try:
self._do_push(notification)
except Exception, e:
print e
self._send_queue.put(notification)
gevent.sleep(5.0)
finally:
if self._send_queue.qsize() < 1 and \
not self._send_queue_cleared.is_set():
self._send_queue_cleared.set()
class GServer(ProtoBufRPCServer):
def __init__(self, host, port, service, poolsize=128):
self.gpool = Pool(poolsize)
self.stop_event = Event()
context = zmq.Context()
self.port = port
self.socket = context.socket(zmq.ROUTER)
self.socket.bind("tcp://%s:%s" % (host, port))
self.service = service
def serve_forever(self,):
while not self.stop_event.is_set():
try:
msg = self.socket.recv_multipart()
except zmq.ZMQError:
if self.socket.closed:
break
raise e
self.gpool.spawn(self.handle_request, msg)
def shutdown(self,):
self.socket.close()
self.stop_event.set()
def handle_request(self, msg):
assert len(msg) == 3
(id_, null, request) = msg
assert null == ''
response = self.handle(request)
self.socket.send_multipart([id_, null, response.SerializeToString()])
def run(ctx, dev):
"""Start the client"""
# create app
app = EthApp(ctx.obj['config'])
if dev:
gevent.get_hub().SYSTEM_ERROR = BaseException
try:
ctx.obj['config']['client_version'] += '/' + os.getlogin()
except:
log.warn("can't get and add login name to client_version")
pass
# register services
for service in services:
assert issubclass(service, BaseService)
if service.name not in app.config['deactivated_services']:
assert service.name not in app.services
service.register_with_app(app)
assert hasattr(app.services, service.name)
# start app
app.start()
# wait for interupt
evt = Event()
gevent.signal(signal.SIGQUIT, evt.set)
gevent.signal(signal.SIGTERM, evt.set)
gevent.signal(signal.SIGINT, evt.set)
evt.wait()
# finally stop
app.stop()
def test_greenlet(self):
queue = JoinableQueue()
requests_done = Event()
g = Greenlet(self._producer, queue, FirstService(), 'Terminator')
h = Greenlet(self._producer, queue, SecondService(), 'Terminator')
i = Greenlet(self._producer, queue, ThirdService(), 'Terminator')
requests = Group()
for request in g, h, i:
requests.add(request)
log.debug('before spawn')
c = spawn(
self._consumer,
done=requests_done,
queue=queue,
)
[r.start() for r in requests]
log.debug('after spawn')
requests.join()
requests_done.set()
log.debug('requests are done')
c.join()
log.debug('consumer is done')
def test_visual_transform(self):
input_data_product_id = self.ctd_plain_input_data_product()
output_data_product_id = self.google_dt_data_product()
dpd = DataProcessDefinition(name='visual transform')
dpd.data_process_type = DataProcessTypeEnum.TRANSFORM
dpd.module = 'ion.processes.data.transforms.viz.google_dt'
dpd.class_name = 'VizTransformGoogleDT'
#--------------------------------------------------------------------------------
# Walk before we base jump
#--------------------------------------------------------------------------------
data_process_definition_id = self.data_process_management.create_data_process_definition(dpd)
self.addCleanup(self.data_process_management.delete_data_process_definition, data_process_definition_id)
data_process_id = self.data_process_management.create_data_process2(data_process_definition_id=data_process_definition_id, in_data_product_ids=[input_data_product_id], out_data_product_ids=[output_data_product_id])
self.addCleanup(self.data_process_management.delete_data_process2,data_process_id)
self.data_process_management.activate_data_process2(data_process_id)
self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id)
validated = Event()
def validation(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
self.assertTrue(rdt['google_dt_components'] is not None)
validated.set()
self.setup_subscriber(output_data_product_id, callback=validation)
self.publish_to_plain_data_product(input_data_product_id)
self.assertTrue(validated.wait(10))
def __init__(self, config, etcd_api, status_reporter, hosts_ipset):
super(_FelixEtcdWatcher, self).__init__(config.ETCD_ADDR, VERSION_DIR)
self._config = config
self._etcd_api = etcd_api
self._status_reporter = status_reporter
self.hosts_ipset = hosts_ipset
# Keep track of the config loaded from etcd so we can spot if it
# changes.
self.last_global_config = None
self.last_host_config = None
self.my_config_dir = dir_for_per_host_config(self._config.HOSTNAME)
# Events triggered by the EtcdAPI Actor to tell us to load the config
# and start polling. These are one-way flags.
self.load_config = Event()
self.begin_polling = Event()
# Event that we trigger once the config is loaded.
self.configured = Event()
# Polling state initialized at poll start time.
self.splitter = None
# Cache of known endpoints, used to resolve deletions of whole
# directory trees.
self.endpoint_ids_per_host = defaultdict(set)
# Next-hop IP addresses of our hosts, if populated in etcd.
self.ipv4_by_hostname = {}
# Register for events when values change.
self._register_paths()
def test_data_process_prime(self):
self.lc_preload()
instrument_data_product_id = self.ctd_instrument_data_product()
derived_data_product_id = self.ctd_derived_data_product()
data_process_id = self.data_process_management.create_data_process2(in_data_product_ids=[instrument_data_product_id], out_data_product_ids=[derived_data_product_id])
self.addCleanup(self.data_process_management.delete_data_process2, data_process_id)
self.data_process_management.activate_data_process2(data_process_id)
self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id)
validated = Event()
def validation(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
np.testing.assert_array_almost_equal(rdt['conductivity_L1'], np.array([42.914]))
np.testing.assert_array_almost_equal(rdt['temp_L1'], np.array([20.]))
np.testing.assert_array_almost_equal(rdt['pressure_L1'], np.array([3.068]))
np.testing.assert_array_almost_equal(rdt['density'], np.array([1021.7144739593881]))
np.testing.assert_array_almost_equal(rdt['salinity'], np.array([30.935132729668283]))
validated.set()
self.setup_subscriber(derived_data_product_id, callback=validation)
self.publish_to_data_product(instrument_data_product_id)
self.assertTrue(validated.wait(10))
def test_actors(self):
input_data_product_id = self.ctd_plain_input_data_product()
output_data_product_id = self.ctd_plain_density()
actor = self.create_density_transform_function()
route = {input_data_product_id: {output_data_product_id: actor}}
config = DotDict()
config.process.routes = route
config.process.params.lat = 45.
config.process.params.lon = -71.
data_process_id = self.data_process_management.create_data_process2(in_data_product_ids=[input_data_product_id], out_data_product_ids=[output_data_product_id], configuration=config)
self.addCleanup(self.data_process_management.delete_data_process2, data_process_id)
self.data_process_management.activate_data_process2(data_process_id)
self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id)
validated = Event()
def validation(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
# The value I use is a double, the value coming back is only a float32 so there's some data loss but it should be precise to the 4th digit
np.testing.assert_array_almost_equal(rdt['density'], np.array([1021.6839775385847]), decimal=4)
validated.set()
self.setup_subscriber(output_data_product_id, callback=validation)
self.publish_to_plain_data_product(input_data_product_id)
self.assertTrue(validated.wait(10))
def test_with_remoting(defer):
random_data = str(random.randint(0, 10000000000))
node1 = Node('localhost:20001', enable_remoting=True)
defer(node1.stop)
node2 = Node('localhost:20002', enable_remoting=True)
defer(node2.stop)
f_src = tempfile.NamedTemporaryFile()
f_src.write(random_data)
f_src.flush()
class Sender(Actor):
def run(self, receiver):
ref = serve_file(f_src.name)
receiver << ref
class Receiver(Actor):
def receive(self, fref):
for _ in range(2):
fetched_path = fref.fetch()
with open(fetched_path) as f_dst:
eq_(random_data, f_dst.read())
received.set()
received = Event()
node2.spawn(Receiver, name='receiver')
receiver_ref = node1.lookup_str('localhost:20002/receiver')
node1.spawn(Sender.using(receiver=receiver_ref))
received.wait()
def test_reentrant(self):
pool = self.klass(1)
result = pool.apply(pool.apply, (lambda a: a + 1, (5, )))
self.assertEqual(result, 6)
evt = Event()
pool.apply_async(evt.set)
evt.wait()
class Worker(object):
'''
子进程运行的代码,通过起一个协程来和主进程通信
包括接受任务分配请求，退出信号(零字节包)，及反馈任务执行进度
然后主协程等待停止信号并中止进程(stop_event用于协程间同步)。
'''
def __init__(self, url):
self.url = url
self.stop_event = Event()
gevent.spawn(self.communicate)
self.stop_event.wait()
print 'worker(%s):will stop' % os.getpid()
def exec_task(self, task):
print 'worker(%s):execute task:%s' % (os.getpid(), task.rstrip('\n'))
def communicate(self):
print 'worker(%s):started' % os.getpid()
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(self.url)
fp = client.makefile()
while True:
line = fp.readline()
if not line:
self.stop_event.set()
break
'单独起一个协程去执行任务，防止通信协程阻塞'
gevent.spawn(self.exec_task, line)
def test_heartbeat_with_listeners(self):
mocklistener = Mock(spec=ProcessRPCServer)
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc)
readyev = Event()
readyev.set()
mocklistener.get_ready_event.return_value = readyev
def fake_listen(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
mocklistener.listen = lambda *a, **kw: fake_listen(listenoutev, listeninev)
p.start()
p.get_ready_event().wait(timeout=5)
p.start_listeners()
listenoutev.wait(timeout=5) # wait for listen loop to start
self.addCleanup(listeninev.set) # makes listen loop fall out on shutdown
self.addCleanup(p.stop)
# now test heartbeat!
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)
def test_qc_events(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_qc_pdict()
stream_def_id = self.pubsub_management.create_stream_definition('qc stream def', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('qc stream', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
config = DotDict()
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id, config=config)
self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id)
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = np.arange(10) * 3
verified = Event()
def verification(event, *args, **kwargs):
self.assertEquals(event.qc_parameter, 'temp_qc')
self.assertEquals(event.temporal_value, 7)
verified.set()
es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=dataset_id, callback=verification, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(verified.wait(10))
def test_ingestion_failover(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
event = Event()
def cb(*args, **kwargs):
event.set()
sub = EventSubscriber(event_type="ExceptionEvent", callback=cb, origin="stream_exception")
sub.start()
self.publish_fake_data(stream_id, route)
self.wait_until_we_have_enough_granules(dataset_id, 40)
file_path = DatasetManagementService._get_coverage_path(dataset_id)
master_file = os.path.join(file_path, '%s_master.hdf5' % dataset_id)
with open(master_file, 'w') as f:
f.write('this will crash HDF')
self.publish_hifi(stream_id, route, 5)
self.assertTrue(event.wait(10))
sub.stop()
def test_execute_advanced_transform(self):
# Runs a transform across L0-L2 with stream definitions including available fields
streams = self.setup_advanced_transform()
in_stream_id, in_stream_def_id = streams[0]
out_stream_id, out_stream_defs_id = streams[1]
validation_event = Event()
def validator(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
if not np.allclose(rdt['rho'], np.array([1001.0055034])):
return
validation_event.set()
self.setup_validator(validator)
in_route = self.pubsub_management.read_stream_route(in_stream_id)
publisher = StandaloneStreamPublisher(in_stream_id, in_route)
outbound_rdt = RecordDictionaryTool(stream_definition_id=in_stream_def_id)
outbound_rdt['time'] = [0]
outbound_rdt['TEMPWAT_L0'] = [280000]
outbound_rdt['CONDWAT_L0'] = [100000]
outbound_rdt['PRESWAT_L0'] = [2789]
outbound_rdt['lat'] = [45]
outbound_rdt['lon'] = [-71]
outbound_granule = outbound_rdt.to_granule()
publisher.publish(outbound_granule)
self.assertTrue(validation_event.wait(2))
class Console(BaseService):
"""A service starting an interactive ipython session when receiving the
SIGSTP signal (e.g. via keyboard shortcut CTRL-Z).
"""
name = 'console'
def __init__(self, app):
super(Console, self).__init__(app)
self.interrupt = Event()
gevent.signal(signal.SIGTSTP, self.interrupt.set)
self.console_locals = []
def start(self):
super(Console, self).start()
self.console_locals = {}
self.console_locals.update(self.app.services)
self.console_locals['app'] = self.app
def _run(self):
while True:
self.interrupt.wait()
IPython.start_ipython(argv=['--gui', 'gevent'], user_ns=self.console_locals)
self.interrupt.clear()
class InputStream(object):
"""
FCGI_STDIN or FCGI_DATA stream.
Uses temporary file to store received data once max_mem bytes
have been received.
"""
def __init__(self, max_mem=1024):
self._file = SpooledTemporaryFile(max_mem)
self._eof_received = Event()
def feed(self, data):
if self._eof_received.is_set():
raise IOError('Feeding file beyond EOF mark')
if not data: # EOF mark
self._file.seek(0)
self._eof_received.set()
else:
self._file.write(data)
def __iter__(self):
self._eof_received.wait()
return iter(self._file)
def read(self, size=-1):
self._eof_received.wait()
return self._file.read(size)
def readlines(self, sizehint=0):
self._eof_received.wait()
return self._file.readlines(sizehint)
@property
def eof_received(self):
return self._eof_received.is_set()
def __init__(self, sessid, manager, config, error_handler=None):
self.manager = weakref.proxy(manager)
self.sessid = sessid
self.session = manager.make_session(sessid) # the session dict, for general developer usage
for qname in QUEUE_NAMES:
setattr(self, qname, manager.make_queue(sessid, qname))
self.hits = 0
self.heartbeats = 0
self.hb_check_timeout = Event()
self.hb_send_timeout = Event()
self.wsgi_app_greenlet = None
self.state = "NEW"
self.connection_established = False
self.ack_callbacks = {}
self.ack_counter = 0
self.request = None
self.environ = None
self.namespaces = {}
self.active_ns = {} # Namespace sessions that were instantiated
self.jobs = []
self.error_handler = default_error_handler
self.config = config
if error_handler is not None:
self.error_handler = error_handler
def test_data_product_subscription(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='ctd parsed')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(data_product=dp, stream_definition_id=stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
subscription_id = self.pubsub_management.create_subscription('validator', data_product_ids=[data_product_id])
self.addCleanup(self.pubsub_management.delete_subscription, subscription_id)
validated = Event()
def validation(msg, route, stream_id):
validated.set()
stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
dp_stream_id = stream_ids.pop()
validator = StandaloneStreamSubscriber('validator', callback=validation)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_management.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, subscription_id)
route = self.pubsub_management.read_stream_route(dp_stream_id)
publisher = StandaloneStreamPublisher(dp_stream_id, route)
publisher.publish('hi')
self.assertTrue(validated.wait(10))
def test_serialize_compatability(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('ctd extended', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('ctd1', 'xp1', stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
sub_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
verified = Event()
def verifier(msg, route, stream_id):
for k,v in msg.record_dictionary.iteritems():
if v is not None:
self.assertIsInstance(v, np.ndarray)
rdt = RecordDictionaryTool.load_from_granule(msg)
for k,v in rdt.iteritems():
self.assertIsInstance(rdt[k], np.ndarray)
self.assertIsInstance(v, np.ndarray)
verified.set()
subscriber = StandaloneStreamSubscriber('sub1', callback=verifier)
subscriber.start()
self.addCleanup(subscriber.stop)
publisher = StandaloneStreamPublisher(stream_id,route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
ph.fill_rdt(rdt,10)
publisher.publish(rdt.to_granule())
self.assertTrue(verified.wait(60))
def run(ctx, dev, nodial, fake):
"""Start the client ( --dev to stop on error)"""
config = ctx.obj['config']
if nodial:
# config['deactivated_services'].append(PeerManager.name)
# config['deactivated_services'].append(NodeDiscovery.name)
config['discovery']['bootstrap_nodes'] = []
config['discovery']['listen_port'] = 29873
config['p2p']['listen_port'] = 29873
config['p2p']['min_peers'] = 0
if fake:
from ethereum import blocks
blocks.GENESIS_DIFFICULTY = 1024
blocks.BLOCK_DIFF_FACTOR = 16
blocks.MIN_GAS_LIMIT = blocks.GENESIS_GAS_LIMIT / 2
# workaround for genesis.json hack
blocks.GENESIS_JSON["difficulty"] = blocks.int_to_hex(blocks.GENESIS_DIFFICULTY)
# create app
app = EthApp(config)
# development mode
if dev:
gevent.get_hub().SYSTEM_ERROR = BaseException
try:
config['client_version'] += '/' + os.getlogin()
except:
log.warn("can't get and add login name to client_version")
pass
# dump config
konfig.dump_config(config)
# register services
for service in services:
assert issubclass(service, BaseService)
if service.name not in app.config['deactivated_services']:
assert service.name not in app.services
service.register_with_app(app)
assert hasattr(app.services, service.name)
unlock_accounts(ctx.obj['unlock'], app.services.accounts, password=ctx.obj['password'])
# start app
log.info('starting')
app.start()
if config['post_app_start_callback'] is not None:
config['post_app_start_callback'](app)
# wait for interrupt
evt = Event()
gevent.signal(signal.SIGQUIT, evt.set)
gevent.signal(signal.SIGTERM, evt.set)
gevent.signal(signal.SIGINT, evt.set)
evt.wait()
# finally stop
app.stop()
def handle():
connection = create_postgresql_connection()
cursor = connection.cursor()
cursor.execute("BEGIN;")
cursor.execute("DELETE FROM core_ratequery;")
cursor.execute("COMMIT;")
cursor.close()
queue = JoinableQueue()
event = Event()
age_ids = age_map(connection).values() + [None]
sex_ids = sex_map(connection).values() + [None]
education_ids = education_map(connection).values() + [None]
province_ids = province_map(connection).values() + [None]
cursor = connection.cursor()
cursor.execute("SELECT DISTINCT cycle FROM core_microdata;");
cycles = [row[0] for row in cursor]
cursor.close()
greenlets = []
for i in range(50):
gv = gevent.spawn(worker, queue, event)
greenlets.append(gv)
combs = itertools.product(age_ids, sex_ids, province_ids, education_ids, cycles)
for c in combs:
queue.put(c)
queue.join()
event.set()
gevent.joinall(greenlets)
def __init__(self):
#
# Dictionary: key is a websocket, value is a name
#
self.ws_dict = {}
#
# Dictionary: key is a name, value is a websocket
#
self.ws_dict_chan = {}
#
# Dictionary: key is a channel name, value is a list of groupDevices
#
self.channels_to_groupDevices = {}
self.uber_client_ws_client = ""
self.uber_client_ws_server = ""
self.WS_PRIVATE_PORT = 8111
self.WS_PORT = 8112
self.uber_client_ready = Event()
self.private_server_ready = Event()
self.uber_server_socket_ready = Event()
self.public_ready = Event()
self.uber_client_subscriptions_queue = Queue()
self.DEBUG_INTERNAL = DEBUG_INTERNAL
self.DEBUG_EXTERNAL = DEBUG_EXTERNAL
self.DEBUG_OVERRIDE = DEBUG_OVERRIDE
self._lock = gevent.lock.RLock()
print("Initialized WearConnectServer")
def evt_user_input(cls, arg):
trans, ilet = arg
evt = Event()
ilet.event = evt
process_msg(('evt_user_input', arg))
evt.wait()
return ilet
class Pinger(Greenlet):
""" Very simple test 'app'
"""
def __init__(self, id):
super(Pinger,self).__init__()
self.event = Event()
self.conn = None
self.id = id
def _run(self):
logger.debug("Pinger starting")
self.conn = connection.AMQPConnection(self)
self.conn.connect()
#self.conn.connection.join()
self.event.wait()
logger.debug("Pinger exiting")
self.amqp.close()
self.conn.close()
def on_connect(self, connection):
self.amqp = PingerAMQPManager(connection, self, self.id)
def handle_message(self, message):
if message.routing_key.endswith('pinger.exit'):
#self.conn.connection.close()
self.event.set()
class C2DMService(object):
def __init__(self, source, email, password):
self.source = source
self.email = email
self.password = password
self._send_queue = Queue()
self._send_queue_cleared = Event()
self.log = logging.getLogger('pulsus.service.c2dm')
def _send_loop(self):
self._send_greenlet = gevent.getcurrent()
try:
self.log.info("C2DM service started")
while True:
notification = self._send_queue.get()
try:
self._do_push(notification)
except Exception, e:
self.log.exception("Error while pushing")
self._send_queue.put(notification)
gevent.sleep(5.0)
finally:
if self._send_queue.qsize() < 1 and \
not self._send_queue_cleared.is_set():
self._send_queue_cleared.set()
class ClusterCoordinator(Service):
port = Setting('cluster_port', default=4440)
def __init__(self, identity, leader=None, cluster=None):
leader = leader or identity
self.server = PeerServer(self, identity)
self.client = PeerClient(self, leader, identity)
self.set = cluster or ObservableSet()
self.promoted = Event()
self.add_service(self.server)
if leader != identity:
self.add_service(self.client)
self.is_leader = False
else:
self.is_leader = True
def wait_for_promotion(self):
self.promoted.wait()
@property
def leader(self):
return self.client.leader
@property
def identity(self):
return self.client.identity
class uWSGIWebSocket(object): # pragma: no cover
"""
This wrapper class provides a uWSGI WebSocket interface that is
compatible with eventlet's implementation.
"""
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
self.environ = environ
uwsgi.websocket_handshake()
self._req_ctx = None
if hasattr(uwsgi, 'request_context'):
# uWSGI >= 2.1.x with support for api access across-greenlets
self._req_ctx = uwsgi.request_context()
else:
# use event and queue for sending messages
from gevent.event import Event
from gevent.queue import Queue
from gevent.select import select
self._event = Event()
self._send_queue = Queue()
# spawn a select greenlet
def select_greenlet_runner(fd, event):
"""Sets event when data becomes available to read on fd."""
while True:
event.set()
try:
select([fd], [], [])[0]
except ValueError:
break
self._select_greenlet = gevent.spawn(select_greenlet_runner,
uwsgi.connection_fd(),
self._event)
self.app(self)
def close(self):
"""Disconnects uWSGI from the client."""
uwsgi.disconnect()
if self._req_ctx is None:
# better kill it here in case wait() is not called again
self._select_greenlet.kill()
self._event.set()
def _send(self, msg):
"""Transmits message either in binary or UTF-8 text mode,
depending on its type."""
if isinstance(msg, six.binary_type):
method = uwsgi.websocket_send_binary
else:
method = uwsgi.websocket_send
if self._req_ctx is not None:
method(msg, request_context=self._req_ctx)
else:
method(msg)
def _decode_received(self, msg):
"""Returns either bytes or str, depending on message type."""
if not isinstance(msg, six.binary_type):
# already decoded - do nothing
return msg
# only decode from utf-8 if message is not binary data
type = six.byte2int(msg[0:1])
if type >= 48: # no binary
return msg.decode('utf-8')
# binary message, don't try to decode
return msg
def send(self, msg):
"""Queues a message for sending. Real transmission is done in
wait method.
Sends directly if uWSGI version is new enough."""
if self._req_ctx is not None:
self._send(msg)
else:
self._send_queue.put(msg)
self._event.set()
def wait(self):
"""Waits and returns received messages.
If running in compatibility mode for older uWSGI versions,
it also sends messages that have been queued by send().
A return value of None means that connection was closed.
This must be called repeatedly. For uWSGI < 2.1.x it must
be called from the main greenlet."""
while True:
if self._req_ctx is not None:
try:
msg = uwsgi.websocket_recv(request_context=self._req_ctx)
except IOError: # connection closed
return None
return self._decode_received(msg)
else:
# we wake up at least every 3 seconds to let uWSGI
# do its ping/ponging
event_set = self._event.wait(timeout=3)
if event_set:
self._event.clear()
# maybe there is something to send
msgs = []
while True:
try:
msgs.append(self._send_queue.get(block=False))
except gevent.queue.Empty:
break
for msg in msgs:
self._send(msg)
# maybe there is something to receive, if not, at least
# ensure uWSGI does its ping/ponging
try:
msg = uwsgi.websocket_recv_nb()
except IOError: # connection closed
self._select_greenlet.kill()
return None
if msg: # message available
return self._decode_received(msg)
def __init__(self):
self.read = []
self.write = []
self.event = Event()
def __init__(self):
self.events = set()
self.event = Event()
#coding=utf8
import gevent
from gevent.event import Event
'''
事件
'''
evt = Event()
def setter():
'''After 3 seconds, wake all threads waiting on the value of evt'''
print('A: Hey wait for me, I have to do something')
gevent.sleep(3)
print("Ok, I'm done")
evt.set()
def waiter():
'''After 3 seconds the get call will unblock'''
print("I'll wait for you")
evt.wait() # blocking
print("It's about time")
def main():
gevent.joinall([
gevent.spawn(setter),
gevent.spawn(waiter),
gevent.spawn(waiter),
gevent.spawn(waiter),
class MonitorNetwork(Jobmanager, NodeMonitorMixin):
one_min_stats = ['work_restarts', 'new_jobs', 'work_pushes']
defaults = config = dict(coinservs=REQUIRED,
diff1=0x0000FFFF00000000000000000000000000000000000000000000000000000000,
hashes_per_share=0xFFFF,
merged=tuple(),
block_poll=0.2,
job_refresh=15,
rpc_ping_int=2,
pow_block_hash=False,
poll=None,
currency=REQUIRED,
algo=REQUIRED,
pool_address='',
signal=None,
payout_drk_mn=True,
max_blockheight=None)
def __init__(self, config):
NodeMonitorMixin.__init__(self)
self._configure(config)
if get_bcaddress_version(self.config['pool_address']) is None:
raise ConfigurationError("No valid pool address configured! Exiting.")
# Since some MonitorNetwork objs are polling and some aren't....
self.gl_methods = ['_monitor_nodes', '_check_new_jobs']
# Aux network monitors (merged mining)
self.auxmons = []
# internal vars
self._last_gbt = {}
self._job_counter = 0 # a unique job ID counter
# Currently active jobs keyed by their unique ID
self.jobs = {}
self.latest_job = None # The last job that was generated
self.new_job = Event()
self.last_signal = 0.0
# general current network stats
self.current_net = dict(difficulty=None,
height=None,
last_block=0.0,
prev_hash=None,
transactions=None,
subsidy=None)
self.block_stats = dict(accepts=0,
rejects=0,
solves=0,
last_solve_height=None,
last_solve_time=None,
last_solve_worker=None)
self.recent_blocks = deque(maxlen=15)
# Run the looping height poller if we aren't getting push notifications
if (not self.config['signal'] and self.config['poll'] is None) or self.config['poll']:
self.gl_methods.append('_poll_height')
@property
def status(self):
""" For display in the http monitor """
ret = dict(net_state=self.current_net,
block_stats=self.block_stats,
last_signal=self.last_signal,
currency=self.config['currency'],
live_coinservers=len(self._live_connections),
down_coinservers=len(self._down_connections),
coinservers={},
job_count=len(self.jobs))
for connection in self._live_connections:
st = connection.status()
st['status'] = 'live'
ret['coinservers'][connection.name] = st
for connection in self._down_connections:
st = connection.status()
st['status'] = 'down'
ret['coinservers'][connection.name] = st
return ret
def start(self):
Jobmanager.start(self)
if self.config['signal']:
self.logger.info("Listening for push block notifs on signal {}"
.format(self.config['signal']))
gevent.signal(self.config['signal'], self.getblocktemplate, signal=True)
# Find desired auxmonitors
self.config['merged'] = set(self.config['merged'])
found_merged = set()
for mon in self.manager.component_types['Jobmanager']:
if mon.key in self.config['merged']:
self.auxmons.append(mon)
found_merged.add(mon.key)
mon.new_job.rawlink(self.new_merged_work)
for monitor in self.config['merged'] - found_merged:
self.logger.error("Unable to locate Auxmonitor(s) '{}'".format(monitor))
def found_block(self, raw_coinbase, address, worker, hash_hex, header, job, start):
""" Submit a valid block (hopefully!) to the RPC servers """
block = hexlify(job.submit_serial(header, raw_coinbase=raw_coinbase))
result = {}
def record_outcome(success):
# If we've already recorded a result, then return
if result:
return
if start:
submission_time = time.time() - start
self.logger.info(
"Recording block submission outcome {} after {}"
.format(success, submission_time))
if success:
self.manager.log_event(
"{name}.block_submission_{curr}:{t}|ms"
.format(name=self.manager.config['procname'],
curr=self.config['currency'],
t=submission_time * 1000))
if success:
self.block_stats['accepts'] += 1
self.recent_blocks.append(
dict(height=job.block_height, timestamp=int(time.time())))
else:
self.block_stats['rejects'] += 1
self.logger.info("{} BLOCK {}:{} REJECTED"
.format(self.config['currency'], hash_hex,
job.block_height))
result.update(dict(
address=address,
height=job.block_height,
total_subsidy=job.total_value,
fees=job.fee_total,
hex_bits=hexlify(job.bits),
hex_hash=hash_hex,
worker=worker,
algo=job.algo,
merged=False,
success=success,
currency=self.config['currency']
))
def submit_block(conn):
retries = 0
while retries < 5:
retries += 1
res = "failed"
try:
res = conn.submitblock(block)
except (CoinRPCException, socket.error, ValueError) as e:
self.logger.info("Block failed to submit to the server {} with submitblock! {}"
.format(conn.name, e))
if getattr(e, 'error', {}).get('code', 0) != -8:
self.logger.error(getattr(e, 'error'), exc_info=True)
try:
res = conn.getblocktemplate({'mode': 'submit', 'data': block})
except (CoinRPCException, socket.error, ValueError) as e:
self.logger.error("Block failed to submit to the server {}!"
.format(conn.name), exc_info=True)
self.logger.error(getattr(e, 'error'))
if res is None:
self.logger.info("{} BLOCK {}:{} accepted by {}"
.format(self.config['currency'], hash_hex,
job.block_height, conn.name))
record_outcome(True)
break # break retry loop if success
else:
self.logger.error(
"Block failed to submit to the server {}, "
"server returned {}!".format(conn.name, res),
exc_info=True)
sleep(1)
self.logger.info("Retry {} for connection {}".format(retries, conn.name))
for tries in xrange(200):
if not self._live_connections:
self.logger.error("No live connections to submit new block to!"
" Retry {} / 200.".format(tries))
sleep(0.1)
continue
gl = []
for conn in self._live_connections:
# spawn a new greenlet for each submission to do them all async.
# lower orphan chance
gl.append(spawn(submit_block, conn))
gevent.joinall(gl)
# If none of the submission threads were successfull then record a
# failure
if not result:
record_outcome(False)
break
self.logger.log(35, "Valid network block identified!")
self.logger.info("New block at height {} with hash {} and subsidy {}"
.format(job.block_height,
hash_hex,
job.total_value))
self.block_stats['solves'] += 1
self.block_stats['last_solve_hash'] = hash_hex
self.block_stats['last_solve_height'] = job.block_height
self.block_stats['last_solve_worker'] = "{}.{}".format(address, worker)
self.block_stats['last_solve_time'] = datetime.datetime.utcnow()
if __debug__:
self.logger.debug("New block hex dump:\n{}".format(block))
self.logger.debug("Coinbase: {}".format(str(job.coinbase.to_dict())))
for trans in job.transactions:
self.logger.debug(str(trans.to_dict()))
# Pass back all the results to the reporter who's waiting
return result
@loop(interval='block_poll')
def _poll_height(self):
try:
height = self.call_rpc('getblockcount')
except RPCException:
return
if self.current_net['height'] != height:
self.logger.info("New block on main network detected with polling")
self.current_net['height'] = height
self.getblocktemplate(new_block=True)
@loop(interval='job_refresh')
def _check_new_jobs(self):
self.getblocktemplate()
def getblocktemplate(self, new_block=False, signal=False):
if signal:
self.last_signal = time.time()
try:
# request local memory pool and load it in
bt = self.call_rpc('getblocktemplate',
{'capabilities': [
'coinbasevalue',
'coinbase/append',
'coinbase',
'generation',
'time',
'transactions/remove',
'prevblock',
]})
except RPCException:
return False
if self._last_gbt.get('height') != bt['height']:
new_block = True
# If this was from a push signal and the
if signal and new_block:
self.logger.info("Push block signal notified us of a new block!")
elif signal:
self.logger.info("Push block signal notified us of a block we "
"already know about!")
return
# generate a new job if we got some new work!
dirty = False
if bt != self._last_gbt:
self._last_gbt = bt
self._last_gbt['update_time'] = time.time()
dirty = True
if new_block or dirty:
# generate a new job and push it if there's a new block on the
# network
self.generate_job(push=new_block, flush=new_block, new_block=new_block)
def new_merged_work(self, event):
self.generate_job(push=True, flush=event.flush, network='aux')
def generate_job(self, push=False, flush=False, new_block=False, network='main'):
""" Creates a new job for miners to work on. Push will trigger an
event that sends new work but doesn't force a restart. If flush is
true a job restart will be triggered. """
# aux monitors will often call this early when not needed at startup
if not self._last_gbt:
self.logger.warn("Cannot generate new job, missing last GBT info")
return
if self.auxmons:
merged_work = {}
auxdata = {}
for auxmon in self.auxmons:
if auxmon.last_work['hash'] is None:
continue
merged_work[auxmon.last_work['chainid']] = dict(
hash=auxmon.last_work['hash'],
target=auxmon.last_work['type']
)
tree, size = bitcoin_data.make_auxpow_tree(merged_work)
mm_hashes = [merged_work.get(tree.get(i), dict(hash=0))['hash']
for i in xrange(size)]
mm_data = '\xfa\xbemm'
mm_data += bitcoin_data.aux_pow_coinbase_type.pack(dict(
merkle_root=bitcoin_data.merkle_hash(mm_hashes),
size=size,
nonce=0,
))
for auxmon in self.auxmons:
if auxmon.last_work['hash'] is None:
continue
data = dict(target=auxmon.last_work['target'],
hash=auxmon.last_work['hash'],
height=auxmon.last_work['height'],
found_block=auxmon.found_block,
index=mm_hashes.index(auxmon.last_work['hash']),
type=auxmon.last_work['type'],
hashes=mm_hashes)
auxdata[auxmon.config['currency']] = data
else:
auxdata = {}
mm_data = None
# here we recalculate the current merkle branch and partial
# coinbases for passing to the mining clients
coinbase = Transaction()
coinbase.version = 2
# create a coinbase input with encoded height and padding for the
# extranonces so script length is accurate
extranonce_length = (self.manager.config['extranonce_size'] +
self.manager.config['extranonce_serv_size'])
coinbase.inputs.append(
Input.coinbase(self._last_gbt['height'],
addtl_push=[mm_data] if mm_data else [],
extra_script_sig=b'\0' * extranonce_length))
# Payout Darkcoin masternodes
mn_enforcement = self._last_gbt.get('enforce_masternode_payments', True)
if (self.config['payout_drk_mn'] is True or mn_enforcement is True) \
and self._last_gbt.get('payee', '') != '':
# Grab the darkcoin payout amount, default to 20%
payout = self._last_gbt.get('payee_amount', self._last_gbt['coinbasevalue'] / 5)
self._last_gbt['coinbasevalue'] -= payout
coinbase.outputs.append(
Output.to_address(payout, self._last_gbt['payee']))
self.logger.info("Paying out masternode at addr {}. Payout {}. Blockval reduced to {}"
.format(self._last_gbt['payee'], payout, self._last_gbt['coinbasevalue']))
# simple output to the proper address and value
coinbase.outputs.append(
Output.to_address(self._last_gbt['coinbasevalue'], self.config['pool_address']))
job_id = hexlify(struct.pack(str("I"), self._job_counter))
bt_obj = BlockTemplate.from_gbt(self._last_gbt,
coinbase,
extranonce_length,
[Transaction(unhexlify(t['data']), fees=t['fee'])
for t in self._last_gbt['transactions']])
# add in our merged mining data
if mm_data:
hashes = [bitcoin_data.hash256(tx.raw) for tx in bt_obj.transactions]
bt_obj.merkle_link = bitcoin_data.calculate_merkle_link([None] + hashes, 0)
bt_obj.merged_data = auxdata
bt_obj.job_id = job_id
bt_obj.diff1 = self.config['diff1']
bt_obj.algo = self.config['algo']
bt_obj.currency = self.config['currency']
bt_obj.pow_block_hash = self.config['pow_block_hash']
bt_obj.block_height = self._last_gbt['height']
bt_obj.acc_shares = set()
bt_obj.flush = flush
bt_obj.found_block = self.found_block
# Push the fresh job to users after updating details
self._job_counter += 1
if flush:
self.jobs.clear()
self.jobs[job_id] = bt_obj
self.latest_job = bt_obj
if push or flush:
self.new_job.job = bt_obj
self.new_job.set()
self.new_job.clear()
self.logger.info("{}: New block template with {:,} trans. "
"Diff {:,.4f}. Subsidy {:,.2f}. Height {:,}. "
"Merged: {}"
.format("FLUSH" if flush else "PUSH",
len(self._last_gbt['transactions']),
bits_to_difficulty(self._last_gbt['bits']),
self._last_gbt['coinbasevalue'] / 100000000.0,
self._last_gbt['height'],
', '.join(auxdata.keys())))
# Stats and notifications now that it's pushed
if flush:
self._incr('work_restarts')
self._incr('work_pushes')
self.logger.info("New {} network block announced! Wiping previous"
" jobs and pushing".format(network))
elif push:
self.logger.info("New {} network block announced, pushing new job!"
.format(network))
self._incr('work_pushes')
if new_block:
hex_bits = hexlify(bt_obj.bits)
self.current_net['difficulty'] = bits_to_difficulty(hex_bits)
self.current_net['subsidy'] = bt_obj.total_value
self.current_net['height'] = bt_obj.block_height - 1
self.current_net['last_block'] = time.time()
self.current_net['prev_hash'] = bt_obj.hashprev_be_hex
self.current_net['transactions'] = len(bt_obj.transactions)
self.manager.log_event(
"{name}.{curr}.difficulty:{diff}|g\n"
"{name}.{curr}.subsidy:{subsidy}|g\n"
"{name}.{curr}.job_generate:{t}|g\n"
"{name}.{curr}.height:{height}|g"
.format(name=self.manager.config['procname'],
curr=self.config['currency'],
diff=self.current_net['difficulty'],
subsidy=bt_obj.total_value,
height=bt_obj.block_height - 1,
t=(time.time() - self._last_gbt['update_time']) * 1000))
self._incr('new_jobs')
class PriorityQueue(object):
"""A priority queue.
It is greenlet-safe, and offers the ability of changing priorities
and removing arbitrary items.
The queue is implemented as a custom min-heap. The priority is a
mix of a discrete priority level and of the timestamp. The
elements of the queue are QueueItems.
"""
PRIORITY_EXTRA_HIGH = 0
PRIORITY_HIGH = 1
PRIORITY_MEDIUM = 2
PRIORITY_LOW = 3
PRIORITY_EXTRA_LOW = 4
def __init__(self):
"""Create a priority queue."""
# The queue: a min-heap whose elements are of the form
# (priority, timestamp, item), where item is the actual data.
self._queue = []
# Reverse lookup for the items in the queue: a dictionary
# associating the index in the queue to each item.
self._reverse = {}
# Event to signal that there are items in the queue.
self._event = Event()
# Index of the next element that will be added to the queue.
self._next_index = 0
def __len__(self):
return len(self._queue)
def _verify(self):
"""Make sure that the internal state of the queue is consistent.
This is used only for testing.
"""
if len(self._queue) != len(self._reverse):
return False
if len(self._queue) != self.length():
return False
if self.empty() != (self.length() == 0):
return False
if self._event.isSet() == self.empty():
return False
for item, idx in iteritems(self._reverse):
if self._queue[idx].item != item:
return False
return True
def __contains__(self, item):
"""Implement the 'in' operator for an item in the queue.
item (QueueItem): an item to search.
return (bool): True if item is in the queue.
"""
return item in self._reverse
def _swap(self, idx1, idx2):
"""Swap two elements in the queue, keeping their reverse
indices up to date.
idx1 (int): the index of the first element.
idx2 (int): the index of the second element.
"""
self._queue[idx1], self._queue[idx2] = \
self._queue[idx2], self._queue[idx1]
self._reverse[self._queue[idx1].item] = idx1
self._reverse[self._queue[idx2].item] = idx2
def _up_heap(self, idx):
"""Take the element in position idx up in the heap until its
position is the right one.
idx (int): the index of the element to lift.
return (int): the new index of the element.
"""
while idx > 0:
parent = (idx - 1) // 2
if self._queue[idx] < self._queue[parent]:
self._swap(parent, idx)
idx = parent
else:
break
return idx
def _down_heap(self, idx):
"""Take the element in position idx down in the heap until its
position is the right one.
idx (int): the index of the element to lower.
return (int): the new index of the element.
"""
last = len(self._queue) - 1
while 2 * idx + 1 <= last:
child = 2 * idx + 1
if 2 * idx + 2 <= last and \
self._queue[2 * idx + 2] < self._queue[child]:
child = 2 * idx + 2
if self._queue[child] < self._queue[idx]:
self._swap(child, idx)
idx = child
else:
break
return idx
def _updown_heap(self, idx):
"""Perform both operations of up_heap and down_heap on an
element.
idx (int): the index of the element to lift.
return (int): the new index of the element.
"""
idx = self._up_heap(idx)
return self._down_heap(idx)
def push(self, item, priority=None, timestamp=None):
"""Push an item in the queue. If timestamp is not specified,
uses the current time.
item (QueueItem): the item to add to the queue.
priority (int|None): the priority of the item, or None for
medium priority.
timestamp (datetime|None): the time of the submission, or None
to use now.
return (bool): false if the element was already in the queue
and was not pushed again, true otherwise..
"""
if item in self._reverse:
return False
if priority is None:
priority = PriorityQueue.PRIORITY_MEDIUM
if timestamp is None:
timestamp = make_datetime()
index = self._next_index
self._next_index += 1
self._queue.append(QueueEntry(item, priority, timestamp, index))
last = len(self._queue) - 1
self._reverse[item] = last
self._up_heap(last)
# Signal to listener greenlets that there might be something.
self._event.set()
return True
def top(self, wait=False):
"""Return the first element in the queue without extracting it.
wait (bool): if True, block until an element is present.
return (QueueEntry): first element in the queue.
raise (LookupError): on empty queue if wait was false.
"""
if not self.empty():
return self._queue[0]
else:
if not wait:
raise LookupError("Empty queue.")
else:
while True:
if self.empty():
self._event.wait()
continue
return self._queue[0]
def pop(self, wait=False):
"""Extract (and return) the first element in the queue.
wait (bool): if True, block until an element is present.
return (QueueEntry): first element in the queue.
raise (LookupError): on empty queue, if wait was false.
"""
top = self.top(wait)
last = len(self._queue) - 1
self._swap(0, last)
del self._reverse[top.item]
del self._queue[last]
# last is 0 when the queue becomes empty.
if last > 0:
self._down_heap(0)
else:
# Signal that there is nothing left for listeners.
self._event.clear()
return top
def remove(self, item):
"""Remove an item from the queue. Raise a KeyError if not present.
item (QueueItem): the item to remove.
return (QueueEntry): the complete entry removed.
raise (KeyError): if item not present.
"""
pos = self._reverse[item]
entry = self._queue[pos]
last = len(self._queue) - 1
self._swap(pos, last)
del self._reverse[item]
del self._queue[last]
if pos != last:
self._updown_heap(pos)
if self.empty():
self._event.clear()
return entry
def set_priority(self, item, priority):
"""Change the priority of an item inside the queue. Raises an
exception if the item is not in the queue.
item (QueueItem): the item whose priority needs to change.
priority (int): the new priority.
raise (LookupError): if item not present.
"""
pos = self._reverse[item]
self._queue[pos].priority = priority
self._updown_heap(pos)
def length(self):
"""Return the number of elements in the queue.
return (int): length of the queue
"""
return len(self._queue)
def empty(self):
"""Return if the queue is empty.
return (bool): is the queue empty?
"""
return self.length() == 0
def get_status(self):
"""Return the content of the queue. Note that the order may be not
correct, but the first element is the one at the top.
return ([QueueEntry]): a list of entries containing the
representation of the item, the priority and the
timestamp.
"""
return [{
'item': entry.item.to_dict(),
'priority': entry.priority,
'timestamp': make_timestamp(entry.timestamp)
} for entry in self._queue]
class WorkerPool(object):
"""This class keeps the state of the workers attached to ES, and
allow the ES to get a usable worker when it needs it.
"""
WORKER_INACTIVE = None
WORKER_DISABLED = "disabled"
# Seconds after which we declare a worker stale.
WORKER_TIMEOUT = timedelta(seconds=600)
def __init__(self, service):
"""service (Service): the EvaluationService using this
WorkerPool.
"""
self._service = service
self._worker = {}
# These dictionary stores data about the workers (identified
# by their shard number). Schedule disabling to True means
# that we are going to disable the worker as soon as possible
# (when it finishes the current operations). The current
# operations are also discarded because we already re-assigned
# it. Ignore is true if the next results coming from the
# worker should be discarded. Operations is the list of
# operations currently executing. Operations to ignore is the
# list of operations to ignore in the next batch of results.
# Type: {int: [ESOperation]}
self._operations = {}
# Type: {int: [ESOperation]}
self._operations_to_ignore = {}
# Type: {int: Datetime|None}
self._start_time = {}
# Type: {int: bool}
self._schedule_disabling = {}
# Type: {int: bool}
self._ignore = {}
# TODO: given the number of pieces data associated to each
# worker, this class could be simplified by creating a new
# WorkerPoolItem class.
# TODO: at the moment race conditions during the periodic
# checks cannot be excluded. A refactoring of this class
# should take that into account.
# A reverse lookup dictionary mapping operations to shards.
# Type: {ESOperation: int}
self._operations_reverse = dict()
# A lock to ensure that the reverse lookup stays in sync with
# the operations lists.
self._operation_lock = gevent.lock.RLock()
# Event set when there are workers available to take jobs. It
# is only guaranteed that if a worker is available, then this
# event is set. In other words, the fact that this event is
# set does not mean that there is a worker available.
self._workers_available_event = Event()
def __len__(self):
return len(self._worker)
def __contains__(self, operation):
return operation in self._operations_reverse
def _remove_operations(self, shard, new_operation):
"""Safely remove operations from a worker, assigning a new status.
shard (int): the worker from which to remove operations.
new_operations (unicode|None): the new operation, which can be
INACTIVE or DISABLED.
"""
with self._operation_lock:
operations = self._operations[shard]
self._operations[shard] = new_operation
if isinstance(operations, list):
for operation in operations:
del self._operations_reverse[operation]
def _add_operations(self, shard, operations):
"""Assigns new operations to a currently inactive worker.
shard (int): shard of the worker.
operations ([ESOperation]) operations to assign to the worker.
"""
if self._operations[shard] != WorkerPool.WORKER_INACTIVE:
raise ValueError("Shard %s is already doing an operation.", shard)
with self._operation_lock:
self._operations[shard] = operations
for operation in operations:
self._operations_reverse[operation] = shard
def wait_for_workers(self):
"""Wait until a worker might be available."""
self._workers_available_event.wait()
def add_worker(self, worker_coord):
"""Add a new worker to the worker pool.
worker_coord (ServiceCoord): the coordinates of the worker.
"""
shard = worker_coord.shard
# Instruct GeventLibrary to connect ES to the Worker.
self._worker[shard] = self._service.connect_to(
worker_coord,
on_connect=self.on_worker_connected)
# And we fill all data.
self._operations[shard] = WorkerPool.WORKER_INACTIVE
self._operations_to_ignore[shard] = []
self._start_time[shard] = None
self._schedule_disabling[shard] = False
self._ignore[shard] = False
self._workers_available_event.set()
logger.debug("Worker %s added.", shard)
def on_worker_connected(self, worker_coord):
"""To be called when a worker comes alive after being
offline. We use this callback to instruct the worker to
precache all files concerning the contest.
worker_coord (ServiceCoord): the coordinates of the worker
that came online.
"""
shard = worker_coord.shard
logger.info("Worker %s online again.", shard)
if self._service.contest_id is not None:
self._worker[shard].precache_files(
contest_id=self._service.contest_id
)
# We don't requeue the operation, because a connection lost
# does not invalidate a potential result given by the worker
# (as the problem was the connection and not the machine on
# which the worker is). But the worker could have been idling,
# so we wake up the consumers.
self._workers_available_event.set()
def acquire_worker(self, operations):
"""Tries to assign an operation to an available worker. If no workers
are available then this returns None, otherwise this returns
the chosen worker.
operations ([ESOperation]): the operations to assign to a worker.
return (int|None): None if no workers are available, the worker
assigned to the operation otherwise.
"""
# We look for an available worker.
try:
shard = self.find_worker(WorkerPool.WORKER_INACTIVE,
require_connection=True,
random_worker=True)
except LookupError:
self._workers_available_event.clear()
return None
# Then we fill the info for future memory.
self._add_operations(shard, operations)
logger.debug("Worker %s acquired.", shard)
self._start_time[shard] = make_datetime()
with SessionGen() as session:
jobs = []
datasets = {}
submissions = {}
user_tests = {}
for operation in operations:
if operation.dataset_id not in datasets:
datasets[operation.dataset_id] = Dataset.get_from_id(
operation.dataset_id, session)
object_ = None
if operation.for_submission():
if operation.object_id not in submissions:
submissions[operation.object_id] = \
Submission.get_from_id(
operation.object_id, session)
object_ = submissions[operation.object_id]
else:
if operation.object_id not in user_tests:
user_tests[operation.object_id] = \
UserTest.get_from_id(operation.object_id, session)
object_ = user_tests[operation.object_id]
logger.info("Asking worker %s to `%s'.", shard, operation)
jobs.append(Job.from_operation(
operation, object_, datasets[operation.dataset_id]))
job_group_dict = JobGroup(jobs).export_to_dict()
self._worker[shard].execute_job_group(
job_group_dict=job_group_dict,
callback=self._service.action_finished,
plus=shard)
return shard
def release_worker(self, shard):
"""To be called by ES when it receives a notification that an
operation finished.
Note: if the worker is scheduled to be disabled, then we
disable it, and notify the ES to discard the outcome obtained
by the worker.
shard (int): the worker to release.
return (bool|[ESOperation]): if boolean, whether the result is
to be ignored; if a list, the list of operation for which
the results should be ignored.
"""
if self._operations[shard] == WorkerPool.WORKER_INACTIVE:
err_msg = "Trying to release worker while it's inactive."
logger.error(err_msg)
raise ValueError(err_msg)
# If the worker has already been disabled, ignore the result
# and keep the worker disabled.
if self._operations[shard] == WorkerPool.WORKER_DISABLED:
return True
ret = self._ignore[shard]
with self._operation_lock:
to_ignore = self._operations_to_ignore[shard]
self._operations_to_ignore[shard] = []
self._start_time[shard] = None
self._ignore[shard] = False
if self._schedule_disabling[shard]:
self._remove_operations(shard, WorkerPool.WORKER_DISABLED)
self._schedule_disabling[shard] = False
logger.info("Worker %s released and disabled.", shard)
else:
self._remove_operations(shard, WorkerPool.WORKER_INACTIVE)
self._workers_available_event.set()
logger.debug("Worker %s released.", shard)
if ret is False and to_ignore != []:
return to_ignore
else:
return ret
def find_worker(self, operation, require_connection=False,
random_worker=False):
"""Return a worker whose assigned operation is operation.
Remember that there is a placeholder operation to signal that the
worker is not doing anything (or disabled).
operation (ESOperation|unicode|None): the operation we are
looking for, or WorkerPool.WORKER_*.
require_connection (bool): True if we want to find a worker
doing the operation and that is actually connected to us
(i.e., did not die).
random_worker (bool): if True, choose uniformly amongst all
workers doing the operation.
returns (int): the shard of a worker working on operation.
raise (LookupError): if nothing has been found.
"""
pool = []
for shard, worker_operation in iteritems(self._operations):
if worker_operation == operation:
if not require_connection or self._worker[shard].connected:
pool.append(shard)
if not random_worker:
return shard
if pool == []:
raise LookupError("No such operation.")
else:
return random.choice(pool)
def ignore_operation(self, operation):
"""Mark the operation to be ignored.
operation (ESOperation): the operation to ignore.
raise (LookupError): if operation is not found.
"""
try:
with self._operation_lock:
shard = self._operations_reverse[operation]
self._operations_to_ignore[shard].append(operation)
except LookupError:
logger.debug("Asked to ignore operation `%s' "
"that cannot be found.", operation)
raise
def get_status(self):
"""Returns a dict with info about the current status of all
workers.
return (dict): dict of info: current operation, starting time,
number of errors, and additional data specified in the
operation.
"""
result = dict()
for shard in iterkeys(self._worker):
s_time = self._start_time[shard]
s_time = make_timestamp(s_time) if s_time is not None else None
result["%d" % shard] = {
'connected': self._worker[shard].connected,
'operations': [operation.to_dict()
for operation in self._operations[shard]]
if isinstance(self._operations[shard], list)
else self._operations[shard],
'start_time': s_time}
return result
def check_timeouts(self):
"""Check if some worker is not responding in too much time. If
this is the case, the worker is scheduled for disabling, and
we send it a message trying to shut it down.
return ([ESOperation]): list of operations assigned to worker
that timeout.
"""
now = make_datetime()
lost_operations = []
for shard in self._worker:
if self._start_time[shard] is not None:
active_for = now - self._start_time[shard]
if active_for > WorkerPool.WORKER_TIMEOUT:
# Here shard is a working worker with no sign of
# intelligent life for too much time.
logger.error("Disabling and shutting down "
"worker %d because of no response "
"in %s.", shard, active_for)
is_busy = (self._operations[shard] !=
WorkerPool.WORKER_INACTIVE and
self._operations[shard] !=
WorkerPool.WORKER_DISABLED)
assert is_busy
# We return the operation so ES can do what it needs.
if not self._ignore[shard] and \
isinstance(self._operations[shard], list):
for operation in self._operations[shard]:
if operation not in \
self._operations_to_ignore[shard]:
lost_operations.append(operation)
# Also, we are not trusting it, so we are not
# assigning it new operations even if it comes back to
# life.
self._schedule_disabling[shard] = True
self._ignore[shard] = True
self.release_worker(shard)
self._worker[shard].quit(
reason="No response in %s." % active_for)
return lost_operations
def disable_worker(self, shard):
"""Disable a worker.
shard (int): which worker to disable.
return ([ESOperation]): list of non-ignored operations
assigned to the worker.
raise (ValueError): if worker is already disabled.
"""
if self._operations[shard] == WorkerPool.WORKER_DISABLED:
err_msg = \
"Trying to disable already disabled worker %s." % shard
logger.warning(err_msg)
raise ValueError(err_msg)
lost_operations = []
if self._operations[shard] == WorkerPool.WORKER_INACTIVE:
self._operations[shard] = WorkerPool.WORKER_DISABLED
else:
# We return all non-ignored operations so ES can do what
# it needs.
if not self._ignore[shard]:
to_ignore = self._operations_to_ignore[shard]
if isinstance(self._operations[shard], list):
for operation in self._operations[shard]:
if operation not in to_ignore:
lost_operations.append(operation)
# And we mark the worker as disabled (until another action
# is taken).
self._schedule_disabling[shard] = True
self._operations_to_ignore[shard] = []
self._ignore[shard] = True
self.release_worker(shard)
logger.info("Worker %s disabled.", shard)
return lost_operations
def enable_worker(self, shard):
"""Enable a worker that previously was disabled.
shard (int): which worker to enable.
raise (ValueError): if worker is not disabled.
"""
if self._operations[shard] != WorkerPool.WORKER_DISABLED:
err_msg = \
"Trying to enable worker %s which is not disabled." % shard
logger.error(err_msg)
raise ValueError(err_msg)
self._operations[shard] = WorkerPool.WORKER_INACTIVE
self._operations_to_ignore[shard] = []
self._workers_available_event.set()
logger.info("Worker %s enabled.", shard)
def check_connections(self):
"""Check if a worker we assigned an operation to disconnects. In this
case, requeue the operation.
return ([ESOperation]): list of operations assigned to worker
that disconnected.
"""
lost_operations = []
for shard in self._worker:
if not self._worker[shard].connected and \
self._operations[shard] not in [
WorkerPool.WORKER_DISABLED,
WorkerPool.WORKER_INACTIVE]:
if not self._ignore[shard]:
lost_operations += self._operations[shard]
self.release_worker(shard)
return lost_operations
def __init__(
self,
chain: BlockChainService,
query_start_block: typing.BlockNumber,
default_registry: TokenNetworkRegistry,
default_secret_registry: SecretRegistry,
private_key_bin,
transport,
config,
discovery=None,
):
if not isinstance(private_key_bin,
bytes) or len(private_key_bin) != 32:
raise ValueError('invalid private_key')
self.tokennetworkids_to_connectionmanagers = dict()
self.identifier_to_results: typing.Dict[typing.PaymentIdentifier,
AsyncResult, ] = dict()
self.chain: BlockChainService = chain
self.default_registry = default_registry
self.query_start_block = query_start_block
self.default_secret_registry = default_secret_registry
self.config = config
self.privkey = private_key_bin
self.address = privatekey_to_address(private_key_bin)
self.discovery = discovery
self.private_key = PrivateKey(private_key_bin)
self.pubkey = self.private_key.public_key.format(compressed=False)
self.transport = transport
self.blockchain_events = BlockchainEvents()
self.alarm = AlarmTask(chain)
self.shutdown_timeout = config['shutdown_timeout']
self.stop_event = Event()
self.start_event = Event()
self.chain.client.inject_stop_event(self.stop_event)
self.wal = None
self.snapshot_group = 0
# This flag will be used to prevent the service from processing
# state changes events until we know that pending transactions
# have been dispatched.
self.dispatch_events_lock = Semaphore(1)
self.database_path = config['database_path']
if self.database_path != ':memory:':
database_dir = os.path.dirname(config['database_path'])
os.makedirs(database_dir, exist_ok=True)
self.database_dir = database_dir
# Prevent concurrent access to the same db
self.lock_file = os.path.join(self.database_dir, '.lock')
self.db_lock = filelock.FileLock(self.lock_file)
else:
self.database_path = ':memory:'
self.database_dir = None
self.lock_file = None
self.serialization_file = None
self.db_lock = None
self.event_poll_lock = gevent.lock.Semaphore()
class EchoNode: # pragma: no unittest
def __init__(self, api: RaidenAPI, token_address: TokenAddress):
assert isinstance(api, RaidenAPI)
self.ready = Event()
self.api = api
self.token_address = token_address
existing_channels = self.api.get_channel_list(
api.raiden.default_registry.address, self.token_address)
open_channels = [
channel_state for channel_state in existing_channels
if channel.get_status(channel_state) == ChannelState.STATE_OPENED
]
if len(open_channels) == 0:
token_proxy = self.api.raiden.proxy_manager.token(
self.token_address)
if not token_proxy.balance_of(self.api.raiden.address) > 0:
raise ValueError(
f"Not enough funds for echo node "
f"{to_checksum_address(self.api.raiden.address)} for token "
f"{to_checksum_address(self.token_address)}")
# Using the balance of the node as funds
funds = TokenAmount(token_proxy.balance_of(
self.api.raiden.address))
self.api.token_network_connect(
registry_address=self.api.raiden.default_registry.address,
token_address=self.token_address,
funds=funds,
initial_channel_target=10,
joinable_funds_target=0.5,
)
self.num_seen_events = 0
self.received_transfers: Queue[EventPaymentReceivedSuccess] = Queue()
self.stop_signal = None # used to signal REMOVE_CALLBACK and stop echo_workers
self.greenlets: Set[Greenlet] = set()
self.lock = BoundedSemaphore()
self.seen_transfers: Deque[EventPaymentReceivedSuccess] = deque(
list(), TRANSFER_MEMORY)
self.num_handled_transfers = 0
self.lottery_pool = Queue()
# register ourselves with the raiden alarm task
self.api.raiden.alarm.register_callback(self.echo_node_alarm_callback)
self.echo_worker_greenlet = gevent.spawn(self.echo_worker)
log.info("Echo node started")
def echo_node_alarm_callback(self, block: Dict[str, Any]):
""" This can be registered with the raiden AlarmTask.
If `EchoNode.stop()` is called, it will give the return signal to be removed from
the AlarmTask callbacks.
"""
if not self.ready.is_set():
self.ready.set()
log.debug(
"echo_node callback",
node=to_checksum_address(self.api.address),
block_number=block["number"],
)
if self.stop_signal is not None:
return REMOVE_CALLBACK
else:
self.greenlets.add(gevent.spawn(self.poll_all_received_events))
return True
def poll_all_received_events(self) -> None:
""" This will be triggered once for each `echo_node_alarm_callback`.
It polls all channels for `EventPaymentReceivedSuccess` events,
adds all new events to the `self.received_transfers` queue and
respawns `self.echo_worker`, if it died. """
locked = False
try:
with Timeout(10):
locked = self.lock.acquire(blocking=False)
if not locked:
return
else:
received_transfers: List[
Event] = self.api.get_raiden_events_payment_history(
token_address=self.token_address,
offset=self.num_seen_events)
received_transfers = [
event for event in received_transfers
if type(event) == EventPaymentReceivedSuccess
]
for event in received_transfers:
transfer = copy.deepcopy(event)
self.received_transfers.put(transfer)
# set last_poll_block after events are enqueued (timeout safe)
if received_transfers:
self.num_seen_events += len(received_transfers)
if not bool(self.echo_worker_greenlet):
log.debug(
"Restarting echo_worker_greenlet",
node=to_checksum_address(self.api.address),
dead=self.echo_worker_greenlet.dead,
successful=self.echo_worker_greenlet.successful(),
exception=self.echo_worker_greenlet.exception,
)
self.echo_worker_greenlet = gevent.spawn(
self.echo_worker)
except Timeout:
log.info("Timeout while polling for events")
finally:
if locked:
self.lock.release()
def echo_worker(self):
""" The `echo_worker` works through the `self.received_transfers` queue and spawns
`self.on_transfer` greenlets for all not-yet-seen transfers. """
log.debug("echo worker", qsize=self.received_transfers.qsize())
while self.stop_signal is None:
if self.received_transfers.qsize() > 0:
transfer = self.received_transfers.get()
if transfer in self.seen_transfers:
log.debug(
"Duplicate transfer ignored",
node=to_checksum_address(self.api.address),
initiator=to_checksum_address(transfer.initiator),
amount=transfer.amount,
identifier=transfer.identifier,
)
else:
self.seen_transfers.append(transfer)
self.greenlets.add(gevent.spawn(self.on_transfer,
transfer))
else:
gevent.sleep(0.5)
def on_transfer(self, transfer):
""" This handles the echo logic, as described in
https://github.com/raiden-network/raiden/issues/651:
- for transfers with an amount that satisfies `amount % 3 == 0`, it sends a transfer
with an amount of `amount - 1` back to the initiator
- for transfers with a "lucky number" amount `amount == 7` it does not send anything
back immediately -- after having received "lucky number transfers" from 7 different
addresses it sends a transfer with `amount = 49` to one randomly chosen one
(from the 7 lucky addresses)
- consecutive entries to the lucky lottery will receive the current pool size as the
`echo_amount`
- for all other transfers it sends a transfer with the same `amount` back to the
initiator
"""
echo_amount = 0
if transfer.amount % 3 == 0:
log.info(
"Received amount divisible by three",
node=to_checksum_address(self.api.address),
initiator=to_checksum_address(transfer.initiator),
amount=transfer.amount,
identifier=transfer.identifier,
)
echo_amount = TokenAmount(transfer.amount - 1)
elif transfer.amount == 7:
log.info(
"Received lottery entry",
node=to_checksum_address(self.api.address),
initiator=to_checksum_address(transfer.initiator),
amount=transfer.amount,
identifier=transfer.identifier,
poolsize=self.lottery_pool.qsize(),
)
# obtain a local copy of the pool
pool = self.lottery_pool.copy()
tickets = [pool.get() for _ in range(pool.qsize())]
assert pool.empty()
del pool
if any(ticket.initiator == transfer.initiator
for ticket in tickets):
assert transfer not in tickets
log.debug(
"Duplicate lottery entry",
node=to_checksum_address(self.api.address),
initiator=to_checksum_address(transfer.initiator),
identifier=transfer.identifier,
poolsize=len(tickets),
)
# signal the poolsize to the participant
echo_amount = len(tickets)
# payout
elif len(tickets) == 6:
log.info("Payout!")
# reset the pool
assert self.lottery_pool.qsize() == 6
self.lottery_pool = Queue()
# add the new participant
tickets.append(transfer)
# choose the winner
transfer = random.choice(tickets)
echo_amount = 49
else:
self.lottery_pool.put(transfer)
else:
log.debug(
"Received transfer",
node=to_checksum_address(self.api.address),
initiator=to_checksum_address(transfer.initiator),
amount=transfer.amount,
identifier=transfer.identifier,
)
echo_amount = transfer.amount
if echo_amount:
echo_identifier = transfer.identifier + echo_amount
log.debug(
"Sending echo transfer",
node=to_checksum_address(self.api.address),
target=to_checksum_address(transfer.initiator),
amount=echo_amount,
original_identifier=transfer.identifier,
echo_identifier=echo_identifier,
token_address=to_checksum_address(self.token_address),
num_handled_transfers=self.num_handled_transfers + 1,
)
self.api.transfer(
registry_address=self.api.raiden.default_registry.address,
token_address=self.token_address,
amount=echo_amount,
target=transfer.initiator,
identifier=echo_identifier,
)
self.num_handled_transfers += 1
def stop(self):
self.stop_signal = True
self.greenlets.add(self.echo_worker_greenlet)
gevent.joinall(self.greenlets, raise_error=True)
import time
import random
from flask import Blueprint, Response
from apps import AppBlueprint
from threading import Thread
from gevent.event import Event, AsyncResult
from gevent import sleep
blueprint = AppBlueprint(blueprint=Blueprint('HelloWorldPage', __name__))
blueprint2 = AppBlueprint(blueprint=Blueprint('HelloWorldPage2', __name__),
rule='/<string:action>')
__sync_signal = Event()
random_event_result = AsyncResult()
def load(*args, **kwargs):
return {}
# These blueprints will be registered with the Flask app and can be used to make your own endpoints.
@blueprint.blueprint.route('/test_blueprint')
def test_basic_blueprint():
# This can be called using the url /apps/HelloWorld/test_blueprint
return 'successfully called basic blueprint'
def random_number_receiver():
while True:
data = random_event_result.get()
yield 'data: %s\n\n' % data
def single_queue_send(
transport: 'UDPTransport',
recipient: Address,
queue: Queue_T,
queue_identifier: QueueIdentifier,
event_stop: Event,
event_healthy: Event,
event_unhealthy: Event,
message_retries: int,
message_retry_timeout: int,
message_retry_max_timeout: int,
):
""" Handles a single message queue for `recipient`.
Notes:
- This task must be the only consumer of queue.
- This task can be killed at any time, but the intended usage is to stop it
with the event_stop.
- If there are many queues for the same recipient, it is the
caller's responsibility to not start them together to avoid congestion.
- This task assumes the endpoint is never cleared after it's first known.
If this assumption changes the code must be updated to handle unknown
addresses.
"""
# A NotifyingQueue is required to implement cancelability, otherwise the
# task cannot be stopped while the greenlet waits for an element to be
# inserted in the queue.
if not isinstance(queue, NotifyingQueue):
raise ValueError('queue must be a NotifyingQueue.')
# Reusing the event, clear must be carefully done
data_or_stop = event_first_of(
queue,
event_stop,
)
# Wait for the endpoint registration or to quit
transport.log.debug(
'queue: waiting for node to become healthy',
queue_identifier=queue_identifier,
queue_size=len(queue),
)
event_first_of(
event_healthy,
event_stop,
).wait()
transport.log.debug(
'queue: processing queue',
queue_identifier=queue_identifier,
queue_size=len(queue),
)
while True:
data_or_stop.wait()
if event_stop.is_set():
transport.log.debug(
'queue: stopping',
queue_identifier=queue_identifier,
queue_size=len(queue),
)
return
# The queue is not empty at this point, so this won't raise Empty.
# This task being the only consumer is a requirement.
(messagedata, message_id) = queue.peek(block=False)
transport.log.debug(
'queue: sending message',
recipient=pex(recipient),
msgid=message_id,
queue_identifier=queue_identifier,
queue_size=len(queue),
)
backoff = timeout_exponential_backoff(
message_retries,
message_retry_timeout,
message_retry_max_timeout,
)
acknowledged = retry_with_recovery(
transport,
messagedata,
message_id,
recipient,
event_stop,
event_healthy,
event_unhealthy,
backoff,
)
if acknowledged:
queue.get()
# Checking the length of the queue does not trigger a
# context-switch, so it's safe to assume the length of the queue
# won't change under our feet and when a new item will be added the
# event will be set again.
if not queue:
data_or_stop.clear()
if event_stop.is_set():
return
class RaidenService:
""" A Raiden node. """
def __init__(
self,
chain: BlockChainService,
query_start_block: typing.BlockNumber,
default_registry: TokenNetworkRegistry,
default_secret_registry: SecretRegistry,
private_key_bin,
transport,
config,
discovery=None,
):
if not isinstance(private_key_bin,
bytes) or len(private_key_bin) != 32:
raise ValueError('invalid private_key')
self.tokennetworkids_to_connectionmanagers = dict()
self.identifier_to_results: typing.Dict[typing.PaymentIdentifier,
AsyncResult, ] = dict()
self.chain: BlockChainService = chain
self.default_registry = default_registry
self.query_start_block = query_start_block
self.default_secret_registry = default_secret_registry
self.config = config
self.privkey = private_key_bin
self.address = privatekey_to_address(private_key_bin)
self.discovery = discovery
self.private_key = PrivateKey(private_key_bin)
self.pubkey = self.private_key.public_key.format(compressed=False)
self.transport = transport
self.blockchain_events = BlockchainEvents()
self.alarm = AlarmTask(chain)
self.shutdown_timeout = config['shutdown_timeout']
self.stop_event = Event()
self.start_event = Event()
self.chain.client.inject_stop_event(self.stop_event)
self.wal = None
self.snapshot_group = 0
# This flag will be used to prevent the service from processing
# state changes events until we know that pending transactions
# have been dispatched.
self.dispatch_events_lock = Semaphore(1)
self.database_path = config['database_path']
if self.database_path != ':memory:':
database_dir = os.path.dirname(config['database_path'])
os.makedirs(database_dir, exist_ok=True)
self.database_dir = database_dir
# Prevent concurrent access to the same db
self.lock_file = os.path.join(self.database_dir, '.lock')
self.db_lock = filelock.FileLock(self.lock_file)
else:
self.database_path = ':memory:'
self.database_dir = None
self.lock_file = None
self.serialization_file = None
self.db_lock = None
self.event_poll_lock = gevent.lock.Semaphore()
def start_async(self) -> Event:
""" Start the node asynchronously. """
self.start_event.clear()
self.stop_event.clear()
if self.database_dir is not None:
self.db_lock.acquire(timeout=0)
assert self.db_lock.is_locked
# start the registration early to speed up the start
if self.config['transport_type'] == 'udp':
endpoint_registration_greenlet = gevent.spawn(
self.discovery.register,
self.address,
self.config['transport']['udp']['external_ip'],
self.config['transport']['udp']['external_port'],
)
# The database may be :memory:
storage = sqlite.SQLiteStorage(self.database_path,
serialize.PickleSerializer())
self.wal = wal.restore_from_latest_snapshot(
node.state_transition,
storage,
)
if self.wal.state_manager.current_state is None:
log.debug('No recoverable state available, created inital state')
block_number = self.chain.block_number()
state_change = ActionInitChain(
random.Random(),
block_number,
self.chain.node_address,
self.chain.network_id,
)
self.wal.log_and_dispatch(state_change, block_number)
payment_network = PaymentNetworkState(
self.default_registry.address,
[], # empty list of token network states as it's the node's startup
)
state_change = ContractReceiveNewPaymentNetwork(
constants.NULL_ADDRESS,
payment_network,
)
self.handle_state_change(state_change)
# On first run Raiden needs to fetch all events for the payment
# network, to reconstruct all token network graphs and find opened
# channels
last_log_block_number = 0
else:
# The `Block` state change is dispatched only after all the events
# for that given block have been processed, filters can be safely
# installed starting from this position without losing events.
last_log_block_number = views.block_number(
self.wal.state_manager.current_state)
log.debug('Restored state from WAL',
last_restored_block=last_log_block_number)
# Restore the current snapshot group
self.snapshot_group = last_log_block_number // SNAPSHOT_BLOCK_COUNT
# Install the filters using the correct from_block value, otherwise
# blockchain logs can be lost.
self.install_all_blockchain_filters(
self.default_registry,
self.default_secret_registry,
last_log_block_number,
)
# Complete the first_run of the alarm task and synchronize with the
# blockchain since the last run.
#
# Notes about setup order:
# - The filters must be polled after the node state has been primed,
# otherwise the state changes won't have effect.
# - The alarm must complete its first run before the transport is started,
# to avoid rejecting messages for unknown channels.
self.alarm.register_callback(self._callback_new_block)
self.alarm.first_run()
chain_state = views.state_from_raiden(self)
# Dispatch pending transactions
pending_transactions = views.get_pending_transactions(chain_state, )
log.debug(
'Processing pending transactions',
num_pending_transactions=len(pending_transactions),
)
with self.dispatch_events_lock:
for transaction in pending_transactions:
on_raiden_event(self, transaction)
self.alarm.start()
queueids_to_queues = views.get_all_messagequeues(chain_state)
self.transport.start(self, queueids_to_queues)
# Health check needs the transport layer
self.start_neighbours_healthcheck()
if self.config['transport_type'] == 'udp':
def set_start_on_registration(_):
self.start_event.set()
endpoint_registration_greenlet.link(set_start_on_registration)
else:
self.start_event.set()
return self.start_event
def start(self) -> Event:
""" Start the node. """
self.start_async().wait()
def start_neighbours_healthcheck(self):
for neighbour in views.all_neighbour_nodes(
self.wal.state_manager.current_state):
if neighbour != ConnectionManager.BOOTSTRAP_ADDR:
self.start_health_check_for(neighbour)
def stop(self):
""" Stop the node. """
# Needs to come before any greenlets joining
self.stop_event.set()
self.transport.stop_and_wait()
self.alarm.stop_async()
wait_for = [self.alarm]
wait_for.extend(getattr(self.transport, 'greenlets', []))
# We need a timeout to prevent an endless loop from trying to
# contact the disconnected client
gevent.wait(wait_for, timeout=self.shutdown_timeout)
# Filters must be uninstalled after the alarm task has stopped. Since
# the events are polled by an alarm task callback, if the filters are
# uninstalled before the alarm task is fully stopped the callback
# `poll_blockchain_events` will fail.
#
# We need a timeout to prevent an endless loop from trying to
# contact the disconnected client
try:
with gevent.Timeout(self.shutdown_timeout):
self.blockchain_events.uninstall_all_event_listeners()
except (gevent.timeout.Timeout, RaidenShuttingDown):
pass
self.blockchain_events.reset()
if self.db_lock is not None:
self.db_lock.release()
def __repr__(self):
return '<{} {}>'.format(self.__class__.__name__, pex(self.address))
def get_block_number(self):
return views.block_number(self.wal.state_manager.current_state)
def handle_state_change(self, state_change, block_number=None):
log.debug('STATE CHANGE',
node=pex(self.address),
state_change=state_change)
if block_number is None:
block_number = self.get_block_number()
# Take a snapshot every SNAPSHOT_BLOCK_COUNT
# TODO: Gather more data about storage requirements
# and update the value to specify how often we need
# capturing a snapshot should take place
new_snapshot_group = block_number // SNAPSHOT_BLOCK_COUNT
if new_snapshot_group > self.snapshot_group:
log.debug(f'Storing snapshot at block: {block_number}')
self.wal.snapshot()
self.snapshot_group = new_snapshot_group
event_list = self.wal.log_and_dispatch(state_change, block_number)
if self.dispatch_events_lock.locked():
return []
for event in event_list:
log.debug('RAIDEN EVENT',
node=pex(self.address),
raiden_event=event)
on_raiden_event(self, event)
return event_list
def set_node_network_state(self, node_address, network_state):
state_change = ActionChangeNodeNetworkState(node_address,
network_state)
self.wal.log_and_dispatch(state_change, self.get_block_number())
def start_health_check_for(self, node_address):
self.transport.start_health_check(node_address)
def _callback_new_block(self, current_block_number, chain_id):
"""Called once a new block is detected by the alarm task.
Note:
This should be called only once per block, otherwise there will be
duplicated `Block` state changes in the log.
Therefore this method should be called only once a new block is
mined with the appropriate block_number argument from the
AlarmTask.
"""
# Raiden relies on blockchain events to update its off-chain state,
# therefore some APIs /used/ to forcefully poll for events.
#
# This was done for APIs which have on-chain side-effects, e.g.
# openning a channel, where polling the event is required to update
# off-chain state to providing a consistent view to the caller, e.g.
# the channel exists after the API call returns.
#
# That pattern introduced a race, because the events are returned only
# once per filter, and this method would be called concurrently by the
# API and the AlarmTask. The following lock is necessary, to ensure the
# expected side-effects are properly applied (introduced by the commit
# 3686b3275ff7c0b669a6d5e2b34109c3bdf1921d)
with self.event_poll_lock:
for event in self.blockchain_events.poll_blockchain_events(
current_block_number):
# These state changes will be procesed with a block_number
# which is /larger/ than the ChainState's block_number.
on_blockchain_event(self, event, current_block_number,
chain_id)
# On restart the Raiden node will re-create the filters with the
# ethereum node. These filters will have the from_block set to the
# value of the latest Block state change. To avoid missing events
# the Block state change is dispatched only after all of the events
# have been processed.
#
# This means on some corner cases a few events may be applied
# twice, this will happen if the node crashed and some events have
# been processed but the Block state change has not been
# dispatched.
state_change = Block(current_block_number)
self.handle_state_change(state_change, current_block_number)
def sign(self, message):
""" Sign message inplace. """
if not isinstance(message, SignedMessage):
raise ValueError('{} is not signable.'.format(repr(message)))
message.sign(self.private_key)
def install_all_blockchain_filters(
self,
token_network_registry_proxy: TokenNetworkRegistry,
secret_registry_proxy: SecretRegistry,
from_block: typing.BlockNumber,
):
with self.event_poll_lock:
node_state = views.state_from_raiden(self)
channels = views.list_all_channelstate(node_state)
token_networks = views.get_token_network_identifiers(
node_state,
token_network_registry_proxy.address,
)
self.blockchain_events.add_token_network_registry_listener(
token_network_registry_proxy,
from_block,
)
self.blockchain_events.add_secret_registry_listener(
secret_registry_proxy,
from_block,
)
for token_network in token_networks:
token_network_proxy = self.chain.token_network(token_network)
self.blockchain_events.add_token_network_listener(
token_network_proxy,
from_block,
)
for channel_state in channels:
channel_proxy = self.chain.payment_channel(
channel_state.token_network_identifier,
channel_state.identifier,
)
self.blockchain_events.add_payment_channel_listener(
channel_proxy,
from_block,
)
def connection_manager_for_token_network(self, token_network_identifier):
if not is_binary_address(token_network_identifier):
raise InvalidAddress('token address is not valid.')
known_token_networks = views.get_token_network_identifiers(
views.state_from_raiden(self),
self.default_registry.address,
)
if token_network_identifier not in known_token_networks:
raise InvalidAddress('token is not registered.')
manager = self.tokennetworkids_to_connectionmanagers.get(
token_network_identifier)
if manager is None:
manager = ConnectionManager(self, token_network_identifier)
self.tokennetworkids_to_connectionmanagers[
token_network_identifier] = manager
return manager
def leave_all_token_networks(self):
state_change = ActionLeaveAllNetworks()
self.wal.log_and_dispatch(state_change, self.get_block_number())
def close_and_settle(self):
log.info('raiden will close and settle all channels now')
self.leave_all_token_networks()
connection_managers = [
cm for cm in self.tokennetworkids_to_connectionmanagers.values()
]
if connection_managers:
waiting.wait_for_settle_all_channels(
self,
self.alarm.sleep_time,
)
def mediated_transfer_async(
self,
token_network_identifier,
amount,
target,
identifier,
):
""" Transfer `amount` between this node and `target`.
This method will start an asynchronous transfer, the transfer might fail
or succeed depending on a couple of factors:
- Existence of a path that can be used, through the usage of direct
or intermediary channels.
- Network speed, making the transfer sufficiently fast so it doesn't
expire.
"""
async_result = self.start_mediated_transfer(
token_network_identifier,
amount,
target,
identifier,
)
return async_result
def direct_transfer_async(self, token_network_identifier, amount, target,
identifier):
""" Do a direct transfer with target.
Direct transfers are non cancellable and non expirable, since these
transfers are a signed balance proof with the transferred amount
incremented.
Because the transfer is non cancellable, there is a level of trust with
the target. After the message is sent the target is effectively paid
and then it is not possible to revert.
The async result will be set to False iff there is no direct channel
with the target or the payer does not have balance to complete the
transfer, otherwise because the transfer is non expirable the async
result *will never be set to False* and if the message is sent it will
hang until the target node acknowledge the message.
This transfer should be used as an optimization, since only two packets
are required to complete the transfer (from the payers perspective),
whereas the mediated transfer requires 6 messages.
"""
self.start_health_check_for(target)
if identifier is None:
identifier = create_default_identifier()
direct_transfer = ActionTransferDirect(
token_network_identifier,
target,
identifier,
amount,
)
self.handle_state_change(direct_transfer)
def start_mediated_transfer(
self,
token_network_identifier,
amount,
target,
identifier,
):
self.start_health_check_for(target)
if identifier is None:
identifier = create_default_identifier()
if identifier in self.identifier_to_results:
return self.identifier_to_results[identifier]
async_result = AsyncResult()
self.identifier_to_results[identifier] = async_result
secret = random_secret()
init_initiator_statechange = initiator_init(
self,
identifier,
amount,
secret,
token_network_identifier,
target,
)
# Dispatch the state change even if there are no routes to create the
# wal entry.
self.handle_state_change(init_initiator_statechange)
return async_result
def mediate_mediated_transfer(self, transfer: LockedTransfer):
init_mediator_statechange = mediator_init(self, transfer)
self.handle_state_change(init_mediator_statechange)
def target_mediated_transfer(self, transfer: LockedTransfer):
self.start_health_check_for(transfer.initiator)
init_target_statechange = target_init(transfer)
self.handle_state_change(init_target_statechange)
class Worker(Greenlet):
"""
This class actually performs all of the grunt work, a worker
is a Greenlet or Thread which are spawned based on the number
of connections defined.
"""
def __init__(self, connection, work_queue, work_tracker):
Greenlet.__init__(self, run=None)
# Store NNTPConnection Object
self._connection = connection
# Our only communication to the outside world
# event is a simple event handler of type Event()
# we only unblock if this is set
self._event = Event()
# We always process as many queue entries as we can
# TODO: This should contain some sort of container object that
# allows us to process many Message-ID, but we need to know
# their group they belong into. We also need their expected
# filename. We want to allow the NZBFiles to force a filename
# optionally, otherwise the yenc= is used instead. A minimum
# of those 3 entries `must` be in this input queue. Perhaps
# we should just use NNTPContent entries? (they don't have
# an article ID though)
self._work_queue = work_queue
# Store our work tracker
self._work_tracker = work_tracker
# our exit flag, it is set externally
self._exit = Event()
def _run(self):
"""
Read from the work_queue, process it using an NNTPRequest object.
"""
# block until we have an event to handle.
# print "Worker %s ready!" % self
while not self._exit.is_set():
# Begin our loop
try:
request = self._work_queue.get()
if request is StopIteration:
# during a close() call (defined below) we force
# a StopIteration into the queue to force an exit
# from a program level
return
if request.is_set():
# Process has been aborted or is no longer needed
continue
except StopIteration:
# Got Exit
return
except EmptyQueueException:
# Nothing available for us
continue
# Mark ourselves busy
self._work_tracker.mark_busy(self)
# If we reach here, we have a request to process
request.run(connection=self._connection)
# Mark ourselves available again
self._work_tracker.mark_available(self)
# Ensure our connection is closed before we exit
self._connection.close()
def __del__(self):
# If Ctrl-C is pressed or we're forced to break earlier then we may
# end up here. Ensure our connection is closed before we exit
self._connection.close()
class CephFSMirrorThrasher(Thrasher, Greenlet):
"""
CephFSMirrorThrasher::
The CephFSMirrorThrasher thrashes cephfs-mirror daemons during execution of other
tasks (workunits, etc).
The config is optional. Many of the config parameters are a maximum value
to use when selecting a random value from a range. The config is a dict
containing some or all of:
cluster: [default: ceph] cluster to thrash
max_thrash: [default: 1] the maximum number of active cephfs-mirror daemons per
cluster will be thrashed at any given time.
min_thrash_delay: [default: 60] minimum number of seconds to delay before
thrashing again.
max_thrash_delay: [default: 120] maximum number of seconds to delay before
thrashing again.
max_revive_delay: [default: 10] maximum number of seconds to delay before
bringing back a thrashed cephfs-mirror daemon.
randomize: [default: true] enables randomization and use the max/min values
seed: [no default] seed the random number generator
Examples::
The following example disables randomization, and uses the max delay
values:
tasks:
- ceph:
- cephfs_mirror_thrash:
randomize: False
max_thrash_delay: 10
"""
def __init__(self, ctx, config, cluster, daemons):
super(CephFSMirrorThrasher, self).__init__()
self.ctx = ctx
self.config = config
self.cluster = cluster
self.daemons = daemons
self.logger = log
self.name = 'thrasher.cephfs_mirror.[{cluster}]'.format(
cluster=cluster)
self.stopping = Event()
self.randomize = bool(self.config.get('randomize', True))
self.max_thrash = int(self.config.get('max_thrash', 1))
self.min_thrash_delay = float(self.config.get('min_thrash_delay', 5.0))
self.max_thrash_delay = float(self.config.get('max_thrash_delay', 10))
self.max_revive_delay = float(self.config.get('max_revive_delay',
15.0))
def _run(self):
try:
self.do_thrash()
except Exception as e:
# See _run exception comment for MDSThrasher
self.set_thrasher_exception(e)
self.logger.exception("exception:")
# Allow successful completion so gevent doesn't see an exception.
# The DaemonWatchdog will observe the error and tear down the test.
def log(self, x):
"""Write data to logger assigned to this CephFSMirrorThrasher"""
self.logger.info(x)
def stop(self):
self.stopping.set()
def do_thrash(self):
"""
Perform the random thrashing action
"""
self.log('starting thrash for cluster {cluster}'.format(
cluster=self.cluster))
stats = {
"kill": 0,
}
while not self.stopping.is_set():
delay = self.max_thrash_delay
if self.randomize:
delay = random.randrange(self.min_thrash_delay,
self.max_thrash_delay)
if delay > 0.0:
self.log('waiting for {delay} secs before thrashing'.format(
delay=delay))
self.stopping.wait(delay)
if self.stopping.is_set():
continue
killed_daemons = []
weight = 1.0 / len(self.daemons)
count = 0
for daemon in self.daemons:
skip = random.uniform(0.0, 1.0)
if weight <= skip:
self.log(
'skipping daemon {label} with skip ({skip}) > weight ({weight})'
.format(label=daemon.id_, skip=skip, weight=weight))
continue
self.log('kill {label}'.format(label=daemon.id_))
try:
daemon.signal(signal.SIGTERM)
except Exception as e:
self.log(f'exception when stopping mirror daemon: {e}')
else:
killed_daemons.append(daemon)
stats['kill'] += 1
# if we've reached max_thrash, we're done
count += 1
if count >= self.max_thrash:
break
if killed_daemons:
# wait for a while before restarting
delay = self.max_revive_delay
if self.randomize:
delay = random.randrange(0.0, self.max_revive_delay)
self.log(
'waiting for {delay} secs before reviving daemons'.format(
delay=delay))
sleep(delay)
for daemon in killed_daemons:
self.log('waiting for {label}'.format(label=daemon.id_))
try:
run.wait([daemon.proc], timeout=600)
except CommandFailedError:
pass
except:
self.log(
'Failed to stop {label}'.format(label=daemon.id_))
try:
# try to capture a core dump
daemon.signal(signal.SIGABRT)
except socket.error:
pass
raise
finally:
daemon.reset()
for daemon in killed_daemons:
self.log('reviving {label}'.format(label=daemon.id_))
daemon.start()
for stat in stats:
self.log("stat['{key}'] = {value}".format(key=stat,
value=stats[stat]))
from .utilities import JSONDecodeError, load_json, dump_json, static_routes, \
generate_endpoint
from .database import Invite, Message, User, get_invite_by_code, \
get_messages_by_timestamp, get_user_by_name, set_invite_by_code, \
set_message, set_user
from typing import Any, Dict, Callable, Union, List, Optional
from datetime import datetime as DateTime
from base64 import b64decode
from re import compile as regex_compile
from os import path
auth_regex = regex_compile(r"^(?:(\w+) )?(.*)$")
token_regex = regex_compile(r"^(\w+):(.*)$")
username_regex = regex_compile(r"[a-z_]{2,32}")
mut_message_event = Event()
def respond_error(job: HTTPJob, message: str, code: Union[str, int] = 400):
content = f'{{"message":"{message}"}}'.encode("utf-8")
headers = {
"Content-Type": "application/json; charset=utf-8",
"Content-Length": str(len(content))
}
job.write_head(code, headers)
job.close_body(content)
def get_authorized_user(auth_or_rq: Union[HTTPJob, Optional[str]]):
"""Gets the authorized user via the request's authorization header. If for any
class UDPTransport(Runnable):
UDP_MAX_MESSAGE_SIZE = 1200
log = log
log_healthcheck = log_healthcheck
def __init__(self, address, discovery, udpsocket, throttle_policy, config):
super().__init__()
# these values are initialized by the start method
self.queueids_to_queues: Dict = dict()
self.raiden: RaidenService
self.message_handler: MessageHandler
self.discovery = discovery
self.config = config
self.address = address
self.retry_interval = config['retry_interval']
self.retries_before_backoff = config['retries_before_backoff']
self.nat_keepalive_retries = config['nat_keepalive_retries']
self.nat_keepalive_timeout = config['nat_keepalive_timeout']
self.nat_invitation_timeout = config['nat_invitation_timeout']
self.event_stop = Event()
self.event_stop.set()
self.greenlets = list()
self.addresses_events = dict()
self.messageids_to_asyncresults = dict()
# Maps the addresses to a dict with the latest nonce (using a dict
# because python integers are immutable)
self.nodeaddresses_to_nonces = dict()
cache = cachetools.TTLCache(
maxsize=50,
ttl=CACHE_TTL,
)
cache_wrapper = cachetools.cached(cache=cache)
self.get_host_port = cache_wrapper(discovery.get)
self.throttle_policy = throttle_policy
pool = gevent.pool.Pool()
self.server = DatagramServer(
udpsocket,
handle=self.receive,
spawn=pool,
)
def start(
self,
raiden_service: RaidenService,
message_handler: MessageHandler,
prev_auth_data: str, # pylint: disable=unused-argument
):
if not self.event_stop.ready():
raise RuntimeError('UDPTransport started while running')
self.event_stop.clear()
self.raiden = raiden_service
self.log = log.bind(node=pex(self.raiden.address))
self.log_healthcheck = log_healthcheck.bind(
node=pex(self.raiden.address))
self.message_handler = message_handler
# server.stop() clears the handle and the pool. Since this may be a
# restart the handle must always be set
self.server.set_handle(self.receive)
pool = gevent.pool.Pool()
self.server.set_spawn(pool)
self.server.start()
self.log.debug('UDP started')
super().start()
def _run(self): # pylint: disable=method-hidden
""" Runnable main method, perform wait on long-running subtasks """
try:
self.event_stop.wait()
except gevent.GreenletExit: # killed without exception
self.event_stop.set()
gevent.killall(self.greenlets) # kill children
raise # re-raise to keep killed status
except Exception:
self.stop() # ensure cleanup and wait on subtasks
raise
def stop(self):
if self.event_stop.ready():
return # double call, happens on normal stop, ignore
self.event_stop.set()
# Stop handling incoming packets, but don't close the socket. The
# socket can only be safely closed after all outgoing tasks are stopped
self.server.stop_accepting()
# Stop processing the outgoing queues
gevent.wait(self.greenlets)
# All outgoing tasks are stopped. Now it's safe to close the socket. At
# this point there might be some incoming message being processed,
# keeping the socket open is not useful for these.
self.server.stop()
# Calling `.close()` on a gevent socket doesn't actually close the underlying os socket
# so we do that ourselves here.
# See: https://github.com/gevent/gevent/blob/master/src/gevent/_socket2.py#L208
# and: https://groups.google.com/forum/#!msg/gevent/Ro8lRra3nH0/ZENgEXrr6M0J
try:
self.server._socket.close() # pylint: disable=protected-access
except socket.error:
pass
# Set all the pending results to False
for async_result in self.messageids_to_asyncresults.values():
async_result.set(False)
self.log.debug('UDP stopped')
del self.log_healthcheck
del self.log
def get_health_events(self, recipient):
""" Starts a healthcheck task for `recipient` and returns a
HealthEvents with locks to react on its current state.
"""
if recipient not in self.addresses_events:
self.start_health_check(recipient)
return self.addresses_events[recipient]
def whitelist(self, address: Address): # pylint: disable=no-self-use,unused-argument
"""Whitelist peer address to receive communications from
This may be called before transport is started, to ensure events generated during
start are handled properly.
PS: udp currently doesn't do whitelisting, method defined for compatibility with matrix
"""
return
def start_health_check(self, recipient):
""" Starts a task for healthchecking `recipient` if there is not
one yet.
It also whitelists the address
"""
if recipient not in self.addresses_events:
self.whitelist(recipient) # noop for now, for compatibility
ping_nonce = self.nodeaddresses_to_nonces.setdefault(
recipient,
{'nonce': 0}, # HACK: Allows the task to mutate the object
)
events = healthcheck.HealthEvents(
event_healthy=Event(),
event_unhealthy=Event(),
)
self.addresses_events[recipient] = events
greenlet_healthcheck = gevent.spawn(
healthcheck.healthcheck,
self,
recipient,
self.event_stop,
events.event_healthy,
events.event_unhealthy,
self.nat_keepalive_retries,
self.nat_keepalive_timeout,
self.nat_invitation_timeout,
ping_nonce,
)
greenlet_healthcheck.name = f'Healthcheck for {pex(recipient)}'
greenlet_healthcheck.link_exception(self.on_error)
self.greenlets.append(greenlet_healthcheck)
def init_queue_for(
self,
queue_identifier: QueueIdentifier,
items: List[QueueItem_T],
) -> NotifyingQueue:
""" Create the queue identified by the queue_identifier
and initialize it with `items`.
"""
recipient = queue_identifier.recipient
queue = self.queueids_to_queues.get(queue_identifier)
assert queue is None
queue = NotifyingQueue(items=items)
self.queueids_to_queues[queue_identifier] = queue
events = self.get_health_events(recipient)
greenlet_queue = gevent.spawn(
single_queue_send,
self,
recipient,
queue,
queue_identifier,
self.event_stop,
events.event_healthy,
events.event_unhealthy,
self.retries_before_backoff,
self.retry_interval,
self.retry_interval * 10,
)
if queue_identifier.channel_identifier == CHANNEL_IDENTIFIER_GLOBAL_QUEUE:
greenlet_queue.name = f'Queue for {pex(recipient)} - global'
else:
greenlet_queue.name = (
f'Queue for {pex(recipient)} - {queue_identifier.channel_identifier}'
)
greenlet_queue.link_exception(self.on_error)
self.greenlets.append(greenlet_queue)
self.log.debug(
'new queue created for',
queue_identifier=queue_identifier,
items_qty=len(items),
)
return queue
def get_queue_for(
self,
queue_identifier: QueueIdentifier,
) -> NotifyingQueue:
""" Return the queue identified by the given queue identifier.
If the queue doesn't exist it will be instantiated.
"""
queue = self.queueids_to_queues.get(queue_identifier)
if queue is None:
items: List[QueueItem_T] = list()
queue = self.init_queue_for(queue_identifier, items)
return queue
def send_async(
self,
queue_identifier: QueueIdentifier,
message: Message,
):
""" Send a new ordered message to recipient.
Messages that use the same `queue_identifier` are ordered.
"""
recipient = queue_identifier.recipient
if not is_binary_address(recipient):
raise ValueError('Invalid address {}'.format(pex(recipient)))
# These are not protocol messages, but transport specific messages
if isinstance(message, (Delivered, Ping, Pong)):
raise ValueError('Do not use send for {} messages'.format(
message.__class__.__name__))
messagedata = message.encode()
if len(messagedata) > self.UDP_MAX_MESSAGE_SIZE:
raise ValueError(
'message size exceeds the maximum {}'.format(
self.UDP_MAX_MESSAGE_SIZE), )
# message identifiers must be unique
message_id = message.message_identifier
# ignore duplicates
if message_id not in self.messageids_to_asyncresults:
self.messageids_to_asyncresults[message_id] = AsyncResult()
queue = self.get_queue_for(queue_identifier)
queue.put((messagedata, message_id))
assert queue.is_set()
self.log.debug(
'Message queued',
queue_identifier=queue_identifier,
queue_size=len(queue),
message=message,
)
def send_global( # pylint: disable=unused-argument
self,
room: str,
message: Message,
) -> None:
""" This method exists only for interface compatibility with MatrixTransport """
self.log.warning('UDP is unable to send global messages. Ignoring')
def maybe_send(self, recipient: Address, message: Message):
""" Send message to recipient if the transport is running. """
if not is_binary_address(recipient):
raise InvalidAddress('Invalid address {}'.format(pex(recipient)))
messagedata = message.encode()
host_port = self.get_host_port(recipient)
self.maybe_sendraw(host_port, messagedata)
def maybe_sendraw_with_result(
self,
recipient: Address,
messagedata: bytes,
message_id: UDPMessageID,
) -> AsyncResult:
""" Send message to recipient if the transport is running.
Returns:
An AsyncResult that will be set once the message is delivered. As
long as the message has not been acknowledged with a Delivered
message the function will return the same AsyncResult.
"""
async_result = self.messageids_to_asyncresults.get(message_id)
if async_result is None:
async_result = AsyncResult()
self.messageids_to_asyncresults[message_id] = async_result
host_port = self.get_host_port(recipient)
self.maybe_sendraw(host_port, messagedata)
return async_result
def maybe_sendraw(self, host_port: Tuple[int, int], messagedata: bytes):
""" Send message to recipient if the transport is running. """
# Don't sleep if timeout is zero, otherwise a context-switch is done
# and the message is delayed, increasing its latency
sleep_timeout = self.throttle_policy.consume(1)
if sleep_timeout:
gevent.sleep(sleep_timeout)
# Check the udp socket is still available before trying to send the
# message. There must be *no context-switches after this test*.
if hasattr(self.server, 'socket'):
self.server.sendto(
messagedata,
host_port,
)
def receive(
self,
messagedata: bytes,
host_port: Tuple[str, int], # pylint: disable=unused-argument
) -> bool:
""" Handle an UDP packet. """
# pylint: disable=unidiomatic-typecheck
if len(messagedata) > self.UDP_MAX_MESSAGE_SIZE:
self.log.warning(
'Invalid message: Packet larger than maximum size',
message=encode_hex(messagedata),
length=len(messagedata),
)
return False
try:
message = decode(messagedata)
except InvalidProtocolMessage as e:
self.log.warning(
'Invalid protocol message',
error=str(e),
message=encode_hex(messagedata),
)
return False
if type(message) == Pong:
assert isinstance(message, Pong), MYPY_ANNOTATION
self.receive_pong(message)
elif type(message) == Ping:
assert isinstance(message, Ping), MYPY_ANNOTATION
self.receive_ping(message)
elif type(message) == Delivered:
assert isinstance(message, Delivered), MYPY_ANNOTATION
self.receive_delivered(message)
elif message is not None:
self.receive_message(message)
else:
self.log.warning(
'Invalid message: Unknown cmdid',
message=encode_hex(messagedata),
)
return False
return True
def receive_message(self, message: Message):
""" Handle a Raiden protocol message.
The protocol requires durability of the messages. The UDP transport
relies on the node's WAL for durability. The message will be converted
to a state change, saved to the WAL, and *processed* before the
durability is confirmed, which is a stronger property than what is
required of any transport.
"""
self.raiden.on_message(message)
# Sending Delivered after the message is decoded and *processed*
# gives a stronger guarantee than what is required from a
# transport.
#
# Alternatives are, from weakest to strongest options:
# - Just save it on disk and asynchronously process the messages
# - Decode it, save to the WAL, and asynchronously process the
# state change
# - Decode it, save to the WAL, and process it (the current
# implementation)
delivered_message = Delivered(
delivered_message_identifier=message.message_identifier)
self.raiden.sign(delivered_message)
self.maybe_send(
message.sender,
delivered_message,
)
def receive_delivered(self, delivered: Delivered):
""" Handle a Delivered message.
The Delivered message is how the UDP transport guarantees persistence
by the partner node. The message itself is not part of the raiden
protocol, but it's required by this transport to provide the required
properties.
"""
self.raiden.on_message(delivered)
message_id = delivered.delivered_message_identifier
async_result = self.raiden.transport.messageids_to_asyncresults.get(
message_id)
# clear the async result, otherwise we have a memory leak
if async_result is not None:
del self.messageids_to_asyncresults[message_id]
async_result.set()
else:
self.log.warn(
'Unknown delivered message received',
message_id=message_id,
)
# Pings and Pongs are used to check the health status of another node. They
# are /not/ part of the raiden protocol, only part of the UDP transport,
# therefore these messages are not forwarded to the message handler.
def receive_ping(self, ping: Ping):
""" Handle a Ping message by answering with a Pong. """
self.log_healthcheck.debug(
'Ping received',
message_id=ping.nonce,
message=ping,
sender=pex(ping.sender),
)
pong = Pong(nonce=ping.nonce)
self.raiden.sign(pong)
try:
self.maybe_send(ping.sender, pong)
except (InvalidAddress, UnknownAddress) as e:
self.log.debug("Couldn't send the `Delivered` message", e=e)
def receive_pong(self, pong: Pong):
""" Handles a Pong message. """
message_id = ('ping', pong.nonce, pong.sender)
async_result = self.messageids_to_asyncresults.get(message_id)
if async_result is not None:
self.log_healthcheck.debug(
'Pong received',
sender=pex(pong.sender),
message_id=pong.nonce,
)
async_result.set(True)
else:
self.log_healthcheck.warn(
'Unknown pong received',
message_id=message_id,
)
def get_ping(self, nonce: Nonce) -> bytes:
""" Returns a signed Ping message.
Note: Ping messages don't have an enforced ordering, so a Ping message
with a higher nonce may be acknowledged first.
"""
message = Ping(
nonce=nonce,
current_protocol_version=constants.PROTOCOL_VERSION,
)
self.raiden.sign(message)
return message.encode()
def set_node_network_state(self, node_address: Address, node_state):
state_change = ActionChangeNodeNetworkState(node_address, node_state)
self.raiden.handle_and_track_state_change(state_change)
class Queue(gqueue.Queue):
'''A subclass of gevent.queue.Queue used to organize communication messaging between Compysition Actors.
Parameters:
name (str):
| The name of this queue. Used in certain actors to determine origin faster than reverse key-value lookup
'''
def __init__(self, name, *args, **kwargs):
super(Queue, self).__init__(*args, **kwargs)
self.name = name
self.__has_content = Event()
self.__has_content.clear()
def get(self, block=False, *args, **kwargs):
'''Gets an element from the queue.'''
try:
element = super(Queue, self).get(block=block, *args, **kwargs)
except gqueue.Empty:
self.__has_content.clear()
raise QueueEmpty("Queue {0} has no waiting events".format(
self.name))
if self.qsize() == 0:
self.__has_content.clear()
return element
def put(self, element, *args, **kwargs):
'''Puts element in queue.'''
try:
super(Queue, self).put(element, *args, **kwargs)
self.__has_content.set()
except gqueue.Full:
#only if block = False or (block = True and timeout not None)
raise QueueFull(message="Queue {0} is full".format(self.name),
queue=self)
def wait_until_content(self):
'''Blocks until at least 1 slot is taken.'''
self.__has_content.wait()
def wait_until_empty(self):
'''Blocks until the queue is completely empty.'''
while not self.__has_content.is_set():
sleep(0)
def wait_until_free(self):
'''Blocks until the queue has at lease 1 free slot.'''
while self.qsize() >= self.maxsize:
sleep(0)
def dump(self, other_queue):
"""**Dump all items on this queue to another queue**"""
try:
while True:
other_queue.put(self.get(block=False))
except QueueEmpty:
pass
def __init__(self, maxsize=None):
from gevent.event import Event
Queue.__init__(self, maxsize)
self.unfinished_tasks = 0
self._cond = Event()
self._cond.set()
class RaidenService:
""" A Raiden node. """
def __init__(
self,
chain: BlockChainService,
default_registry: Registry,
default_secret_registry: SecretRegistry,
private_key_bin,
transport,
config,
discovery=None,
):
if not isinstance(private_key_bin,
bytes) or len(private_key_bin) != 32:
raise ValueError('invalid private_key')
invalid_timeout = (
config['settle_timeout'] < NETTINGCHANNEL_SETTLE_TIMEOUT_MIN
or config['settle_timeout'] > NETTINGCHANNEL_SETTLE_TIMEOUT_MAX)
if invalid_timeout:
raise ValueError('settle_timeout must be in range [{}, {}]'.format(
NETTINGCHANNEL_SETTLE_TIMEOUT_MIN,
NETTINGCHANNEL_SETTLE_TIMEOUT_MAX,
))
self.tokens_to_connectionmanagers = dict()
self.identifier_to_results = defaultdict(list)
self.chain: BlockChainService = chain
self.default_registry = default_registry
self.default_secret_registry = default_secret_registry
self.config = config
self.privkey = private_key_bin
self.address = privatekey_to_address(private_key_bin)
if config['transport_type'] == 'udp':
endpoint_registration_event = gevent.spawn(
discovery.register,
self.address,
config['external_ip'],
config['external_port'],
)
endpoint_registration_event.link_exception(
endpoint_registry_exception_handler)
self.private_key = PrivateKey(private_key_bin)
self.pubkey = self.private_key.public_key.format(compressed=False)
self.transport = transport
self.blockchain_events = BlockchainEvents()
self.alarm = AlarmTask(chain)
self.shutdown_timeout = config['shutdown_timeout']
self.stop_event = Event()
self.start_event = Event()
self.chain.client.inject_stop_event(self.stop_event)
self.wal = None
self.database_path = config['database_path']
if self.database_path != ':memory:':
database_dir = os.path.dirname(config['database_path'])
os.makedirs(database_dir, exist_ok=True)
self.database_dir = database_dir
# Prevent concurrent acces to the same db
self.lock_file = os.path.join(self.database_dir, '.lock')
self.db_lock = filelock.FileLock(self.lock_file)
else:
self.database_path = ':memory:'
self.database_dir = None
self.lock_file = None
self.serialization_file = None
self.db_lock = None
if config['transport_type'] == 'udp':
# If the endpoint registration fails the node will quit, this must
# finish before starting the transport
endpoint_registration_event.join()
self.event_poll_lock = gevent.lock.Semaphore()
self.start()
def start(self):
""" Start the node. """
if self.stop_event and self.stop_event.is_set():
self.stop_event.clear()
if self.database_dir is not None:
self.db_lock.acquire(timeout=0)
assert self.db_lock.is_locked
# The database may be :memory:
storage = sqlite.SQLiteStorage(self.database_path,
serialize.PickleSerializer())
self.wal, unapplied_events = wal.restore_from_latest_snapshot(
node.state_transition,
storage,
)
if self.wal.state_manager.current_state is None:
block_number = self.chain.block_number()
state_change = ActionInitNode(
random.Random(),
block_number,
)
self.wal.log_and_dispatch(state_change, block_number)
# On first run Raiden needs to fetch all events for the payment
# network, to reconstruct all token network graphs and find opened
# channels
last_log_block_number = None
else:
# The `Block` state change is dispatched only after all the events
# for that given block have been processed, filters can be safely
# installed starting from this position without losing events.
last_log_block_number = views.block_number(
self.wal.state_manager.current_state)
# The time the alarm task is started or the callbacks are installed doesn't
# really matter.
#
# But it is of paramount importance to:
# - Install the filters which will be polled by poll_blockchain_events
# after the state has been primed, otherwise the state changes won't
# have effect.
# - Install the filters using the correct from_block value, otherwise
# blockchain logs can be lost.
self.alarm.register_callback(self._callback_new_block)
self.alarm.start()
self.install_payment_network_filters(
self.default_registry.address,
last_log_block_number,
)
# Start the transport after the registry is queried to avoid warning
# about unknown channels.
queueids_to_queues = views.get_all_messagequeues(
views.state_from_raiden(self))
self.transport.start(self, queueids_to_queues)
# Health check needs the transport layer
self.start_neighbours_healthcheck()
for event in unapplied_events:
on_raiden_event(self, event)
self.start_event.set()
def start_neighbours_healthcheck(self):
for neighbour in views.all_neighbour_nodes(
self.wal.state_manager.current_state):
if neighbour != ConnectionManager.BOOTSTRAP_ADDR:
self.start_health_check_for(neighbour)
def stop(self):
""" Stop the node. """
# Needs to come before any greenlets joining
self.stop_event.set()
self.transport.stop_and_wait()
self.alarm.stop_async()
wait_for = [self.alarm]
wait_for.extend(getattr(self.transport, 'greenlets', []))
# We need a timeout to prevent an endless loop from trying to
# contact the disconnected client
gevent.wait(wait_for, timeout=self.shutdown_timeout)
# Filters must be uninstalled after the alarm task has stopped. Since
# the events are polled by an alarm task callback, if the filters are
# uninstalled before the alarm task is fully stopped the callback
# `poll_blockchain_events` will fail.
#
# We need a timeout to prevent an endless loop from trying to
# contact the disconnected client
try:
with gevent.Timeout(self.shutdown_timeout):
self.blockchain_events.uninstall_all_event_listeners()
except (gevent.timeout.Timeout, RaidenShuttingDown):
pass
if self.db_lock is not None:
self.db_lock.release()
def __repr__(self):
return '<{} {}>'.format(self.__class__.__name__, pex(self.address))
def get_block_number(self):
return views.block_number(self.wal.state_manager.current_state)
def handle_state_change(self, state_change, block_number=None):
log.debug('STATE CHANGE',
node=pex(self.address),
state_change=state_change)
if block_number is None:
block_number = self.get_block_number()
event_list = self.wal.log_and_dispatch(state_change, block_number)
for event in event_list:
log.debug('EVENT', node=pex(self.address), chain_event=event)
on_raiden_event(self, event)
return event_list
def set_node_network_state(self, node_address, network_state):
state_change = ActionChangeNodeNetworkState(node_address,
network_state)
self.wal.log_and_dispatch(state_change, self.get_block_number())
def start_health_check_for(self, node_address):
self.transport.start_health_check(node_address)
def _callback_new_block(self, current_block_number):
"""Called once a new block is detected by the alarm task.
Note:
This should be called only once per block, otherwise there will be
duplicated `Block` state changes in the log.
Therefore this method should be called only once a new block is
mined with the appropriate block_number argument from the
AlarmTask.
"""
# Raiden relies on blockchain events to update its off-chain state,
# therefore some APIs /used/ to forcefully poll for events.
#
# This was done for APIs which have on-chain side-effects, e.g.
# openning a channel, where polling the event is required to update
# off-chain state to providing a consistent view to the caller, e.g.
# the channel exists after the API call returns.
#
# That pattern introduced a race, because the events are returned only
# once per filter, and this method would be called concurrently by the
# API and the AlarmTask. The following lock is necessary, to ensure the
# expected side-effects are properly applied (introduced by the commit
# 3686b3275ff7c0b669a6d5e2b34109c3bdf1921d)
with self.event_poll_lock:
for event in self.blockchain_events.poll_blockchain_events():
# These state changes will be procesed with a block_number
# which is /larger/ than the NodeState's block_number.
on_blockchain_event(self, event, current_block_number)
# On restart the Raiden node will re-create the filters with the
# ethereum node. These filters will have the from_block set to the
# value of the latest Block state change. To avoid missing events
# the Block state change is dispatched only after all of the events
# have been processed.
#
# This means on some corner cases a few events may be applied
# twice, this will happen if the node crashed and some events have
# been processed but the Block state change has not been
# dispatched.
state_change = Block(current_block_number)
self.handle_state_change(state_change, current_block_number)
def sign(self, message):
""" Sign message inplace. """
if not isinstance(message, SignedMessage):
raise ValueError('{} is not signable.'.format(repr(message)))
message.sign(self.private_key, self.address)
def install_payment_network_filters(self,
payment_network_id,
from_block=None):
proxies = get_relevant_proxies(
self.chain,
self.address,
payment_network_id,
)
# Install the filters first to avoid missing changes, as a consequence
# some events might be applied twice.
self.blockchain_events.add_proxies_listeners(proxies, from_block)
token_network_list = list()
for manager in proxies.channel_managers:
manager_address = manager.address
netting_channel_proxies = proxies.channelmanager_nettingchannels[
manager_address]
network = get_token_network_state_from_proxies(
self, manager, netting_channel_proxies)
token_network_list.append(network)
payment_network = PaymentNetworkState(
payment_network_id,
token_network_list,
)
state_change = ContractReceiveNewPaymentNetwork(payment_network)
self.handle_state_change(state_change)
def connection_manager_for_token(self, registry_address, token_address):
if not is_binary_address(token_address):
raise InvalidAddress('token address is not valid.')
known_token_networks = views.get_token_network_addresses_for(
self.wal.state_manager.current_state,
registry_address,
)
if token_address not in known_token_networks:
raise InvalidAddress('token is not registered.')
manager = self.tokens_to_connectionmanagers.get(token_address)
if manager is None:
manager = ConnectionManager(self, registry_address, token_address)
self.tokens_to_connectionmanagers[token_address] = manager
return manager
def leave_all_token_networks(self):
state_change = ActionLeaveAllNetworks()
self.wal.log_and_dispatch(state_change, self.get_block_number())
def close_and_settle(self):
log.info('raiden will close and settle all channels now')
self.leave_all_token_networks()
connection_managers = [
self.tokens_to_connectionmanagers[token_address]
for token_address in self.tokens_to_connectionmanagers
]
if connection_managers:
waiting.wait_for_settle_all_channels(
self,
self.alarm.wait_time,
)
def mediated_transfer_async(
self,
token_network_identifier,
amount,
target,
identifier,
):
""" Transfer `amount` between this node and `target`.
This method will start an asyncronous transfer, the transfer might fail
or succeed depending on a couple of factors:
- Existence of a path that can be used, through the usage of direct
or intermediary channels.
- Network speed, making the transfer sufficiently fast so it doesn't
expire.
"""
async_result = self.start_mediated_transfer(
token_network_identifier,
amount,
target,
identifier,
)
return async_result
def direct_transfer_async(self, token_network_identifier, amount, target,
identifier):
""" Do a direct transfer with target.
Direct transfers are non cancellable and non expirable, since these
transfers are a signed balance proof with the transferred amount
incremented.
Because the transfer is non cancellable, there is a level of trust with
the target. After the message is sent the target is effectively paid
and then it is not possible to revert.
The async result will be set to False iff there is no direct channel
with the target or the payer does not have balance to complete the
transfer, otherwise because the transfer is non expirable the async
result *will never be set to False* and if the message is sent it will
hang until the target node acknowledge the message.
This transfer should be used as an optimization, since only two packets
are required to complete the transfer (from the payers perspective),
whereas the mediated transfer requires 6 messages.
"""
self.transport.start_health_check(target)
if identifier is None:
identifier = create_default_identifier()
direct_transfer = ActionTransferDirect(
token_network_identifier,
target,
identifier,
amount,
)
self.handle_state_change(direct_transfer)
def start_mediated_transfer(
self,
token_network_identifier,
amount,
target,
identifier,
):
self.transport.start_health_check(target)
if identifier is None:
identifier = create_default_identifier()
assert identifier not in self.identifier_to_results
async_result = AsyncResult()
self.identifier_to_results[identifier].append(async_result)
secret = random_secret()
init_initiator_statechange = initiator_init(
self,
identifier,
amount,
secret,
token_network_identifier,
target,
)
# TODO: implement the network timeout raiden.config['msg_timeout'] and
# cancel the current transfer if it happens (issue #374)
#
# Dispatch the state change even if there are no routes to create the
# wal entry.
self.handle_state_change(init_initiator_statechange)
return async_result
def mediate_mediated_transfer(self, transfer: LockedTransfer):
init_mediator_statechange = mediator_init(self, transfer)
self.handle_state_change(init_mediator_statechange)
def target_mediated_transfer(self, transfer: LockedTransfer):
init_target_statechange = target_init(transfer)
self.handle_state_change(init_target_statechange)
def __init__(self, name, *args, **kwargs):
super(Queue, self).__init__(*args, **kwargs)
self.name = name
self.__has_content = Event()
self.__has_content.clear()
class AceClient(object):
def __init__(self, acehost, aceAPIport, aceHTTPport, acehostslist, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = AceConfig.videoseekback
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._lock = threading.Condition(threading.Lock())
self._streamReaderConnection = None
self._streamReaderState = None
self._streamReaderQueue = deque()
self._engine_version_code = 0;
# Logger
logger = logging.getLogger('AceClientimport tracebacknt_init')
# Try to connect AceStream engine
try:
self._socket = telnetlib.Telnet(acehost, aceAPIport, connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = acehost, aceAPIport, aceHTTPport
logger.debug("Successfully connected to AceStream on %s:%d" % (acehost, aceAPIport))
except:
for AceEngine in acehostslist:
try:
self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1], connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[0], AceEngine[1], AceEngine[2]
logger.debug("Successfully connected to AceStream on %s:%d" % (AceEngine[0], AceEngine[1]))
break
except:
logger.debug("The are no alive AceStream on %s:%d" % (AceEngine[0], AceEngine[1]))
pass
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
def __del__(self):
# Destructor just calls destroy() method
self.destroy()
def destroy(self):
'''
AceClient Destructor
'''
if self._shuttingDown.isSet():
# Already in the middle of destroying
return
# Logger
logger = logging.getLogger("AceClient_destroy")
# We should resume video to prevent read greenlet deadlock
self._resumeevent.set()
# And to prevent getUrl deadlock
self._urlresult.set()
# Trying to disconnect
try:
logger.debug("Destroying client...")
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
# Ignore exceptions on destroy
pass
finally:
self._shuttingDown.set()
def reset(self):
self._started_again = False
self._idleSince = time.time()
self._streamReaderState = None
def _write(self, message):
try:
logger = logging.getLogger("AceClient_write")
logger.debug(message)
self._socket.write(message + "\r\n")
except EOFError as e:
raise AceException("Write error! " + repr(e))
def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, seekback=0):
self._product_key = product_key
self._gender = gender
self._age = age
# Seekback seconds
self._seekback = seekback
# Logger
logger = logging.getLogger("AceClient_aceInit")
# Sending HELLO
self._write(AceMessage.request.HELLO)
if not self._authevent.wait(self._resulttimeout):
errmsg = "Authentication timeout. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = "Authentication error. Wrong key?"
logger.error(errmsg)
raise AceException(errmsg)
return
logger.debug("aceInit ended")
def _getResult(self):
# Logger
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
raise AceException("Result not received")
except gevent.Timeout:
raise AceException("Timeout")
return result
def START(self, datatype, value):
'''
Start video method
'''
if self._engine_version_code >= 3010500 and AceConfig.vlcuse:
stream_type = 'output_format=hls' + ' transcode_audio=' + str(AceConfig.transcode_audio) \
+ ' transcode_mp3=' + str(AceConfig.transcode_mp3) \
+ ' transcode_ac3=' + str(AceConfig.transcode_ac3) \
+ ' preferred_audio_language=' + AceConfig.preferred_audio_language
else:
stream_type = 'output_format=http'
self._urlresult = AsyncResult()
self._write(AceMessage.request.START(datatype.upper(), value, stream_type))
self._getResult()
def STOP(self):
'''
Stop video method
'''
if self._state is not None and self._state != '0':
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult()
def LOADASYNC(self, datatype, value):
self._result = AsyncResult()
self._write(AceMessage.request.LOADASYNC(datatype.upper(), random.randint(1, AceConfig.maxconns * 10000), value))
return self._getResult()
def GETCONTENTINFO(self, datatype, value):
dict = {'torrent': 'url', 'infohash':'infohash', 'raw':'data', 'pid':'content_id'}
self.paramsdict={dict[datatype]:value,'developer_id':'0','affiliate_id':'0','zone_id':'0'}
contentinfo = self.LOADASYNC(datatype, self.paramsdict)
return contentinfo
def GETCID(self, datatype, url):
contentinfo = self.GETCONTENTINFO(datatype, url)
self._cidresult = AsyncResult()
self._write(AceMessage.request.GETCID(contentinfo.get('checksum'), contentinfo.get('infohash'), 0, 0, 0))
cid = self._cidresult.get(True, 5)
return '' if not cid or cid == '' else cid[2:]
def getUrl(self, timeout=30):
# Logger
logger = logging.getLogger("AceClient_getURL")
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = "Engine response time exceeded. GetURL timeout!"
logger.error(errmsg)
raise AceException(errmsg)
def startStreamReader(self, url, cid, counter, req_headers=None):
logger = logging.getLogger("StreamReader")
logger.debug("Opening video stream: %s" % url)
logger.debug("Get headers from client: %s" % req_headers)
self._streamReaderState = 1
try:
if url.endswith('.m3u8'):
logger.debug("Can't stream HLS in non VLC mode: %s" % url)
return None
# Sending client hedaers to AceEngine
if req_headers.has_key('range'):
del req_headers['range']
connection = self._streamReaderConnection = requests.get(url, headers=req_headers, stream=True)
if connection.status_code not in (200, 206):
logger.error("Failed to open video stream %s" % url)
return None
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
while True:
data = None
clients = counter.getClients(cid)
try:
data = connection.raw.read(AceConfig.readchunksize)
except:
break;
if data and clients:
with self._lock:
if len(self._streamReaderQueue) == AceConfig.readchunksize:
self._streamReaderQueue.popleft()
self._streamReaderQueue.append(data)
for c in clients:
try:
c.addChunk(data, 5.0)
except Queue.Full:
if len(clients) > 1:
logger.debug("Disconnecting client: %s" % str(c))
c.destroy()
elif not clients:
logger.debug("All clients disconnected - closing video stream")
break
else:
logger.warning("No data received")
break
except requests.exceptions.ConnectionError:
logger.error("Failed to open video stream")
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
if counter.getClients(cid):
logger.error("Failed to read video stream")
finally:
self.closeStreamReader()
with self._lock:
self._streamReaderState = 3
self._lock.notifyAll()
counter.deleteAll(cid)
def closeStreamReader(self):
logger = logging.getLogger("StreamReader")
c = self._streamReaderConnection
if c:
self._streamReaderConnection = None
c.close()
logger.debug("Video stream closed")
self._streamReaderQueue.clear()
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
def pause(self):
self._write(AceMessage.request.PAUSE)
def play(self):
self._write(AceMessage.request.PLAY)
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while True:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until("\r\n")
self._recvbuffer = self._recvbuffer.strip()
logger.debug('<<< ' + self._recvbuffer)
except:
# If something happened during read, abandon reader.
if not self._shuttingDown.isSet():
logger.error("Exception at socket read")
self._shuttingDown.set()
return
if self._recvbuffer:
# Parsing everything only if the string is not empty
if self._recvbuffer.startswith(AceMessage.response.HELLO):
# Parse HELLO
if 'version_code=' in self._recvbuffer:
v = self._recvbuffer.find('version_code=')
self._engine_version_code = int(self._recvbuffer[v+13:v+20])
if 'key=' in self._recvbuffer:
self._request_key_begin = self._recvbuffer.find('key=')
self._request_key = \
self._recvbuffer[self._request_key_begin + 4:self._request_key_begin + 14]
try:
self._write(AceMessage.request.READY_key(
self._request_key, self._product_key))
except requests.exceptions.ConnectionError as e:
logger.error("Can't connect to keygen server! " + \
repr(e))
self._auth = False
self._authevent.set()
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
# NOTREADY
logger.error("Ace is not ready. Wrong auth?")
self._auth = False
self._authevent.set()
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
# LOADRESP
_contentinfo_raw = self._recvbuffer.split()[2:]
_contentinfo_raw = ' '.join(_contentinfo_raw)
_contentinfo = json.loads(_contentinfo_raw)
if _contentinfo.get('status') == 100:
logger.error("LOADASYNC returned error with message: %s"
% _contentinfo.get('message'))
self._result.set(False)
else:
logger.debug("Content info: %s", _contentinfo)
self._result.set(_contentinfo)
elif self._recvbuffer.startswith(AceMessage.response.START):
# START
if not self._seekback or self._started_again or not self._recvbuffer.endswith(' stream=1'):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seekback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._url = self._recvbuffer.split()[1]
self._urlresult.set(self._url)
self._resumeevent.set()
except IndexError as e:
self._url = None
else:
logger.debug("START received. Waiting for %s." % AceMessage.response.LIVEPOS)
# STOP
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
# SHUTDOWN
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
logger.debug("Got SHUTDOWN from engine")
self._socket.close()
return
# AUTH
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
# Send USERDATA here
self._write(
AceMessage.request.USERDATA(self._gender, self._age))
except:
pass
self._authevent.set()
# GETUSERDATA
elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA):
raise AceException("You should init me first!")
# LIVEPOS
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
self._position = self._recvbuffer.split()
self._position_last = self._position[2].split('=')[1]
self._position_buf = self._position[9].split('=')[1]
self._position = self._position[4].split('=')[1]
if self._seekback and not self._started_again:
self._write(AceMessage.request.LIVESEEK(str(int(self._position_last) - self._seekback)))
logger.debug('Seeking back')
self._started_again = True
# DOWNLOADSTOP
elif self._recvbuffer.startswith(AceMessage.response.DOWNLOADSTOP):
self._state = self._recvbuffer.split()[1]
# STATE
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
# STATUS
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._tempstatus = self._recvbuffer.split()[1].split(';')[0]
if self._tempstatus != self._status:
self._status = self._tempstatus
logger.debug("STATUS changed to " + self._status)
if self._status == 'main:err':
logger.error(
self._status + ' with message ' + self._recvbuffer.split(';')[2])
self._result.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
self._urlresult.set_exception(
AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2]))
elif self._status == 'main:starting':
self._result.set(True)
elif self._status == 'main:idle':
self._result.set(True)
# PAUSE
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
logger.debug("PAUSE event")
self._resumeevent.clear()
# RESUME
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
logger.debug("RESUME event")
gevent.sleep() # PAUSE/RESUME delay
self._resumeevent.set()
# CID
elif self._recvbuffer.startswith('##') or len(self._recvbuffer) == 0:
self._cidresult.set(self._recvbuffer)
logger.debug("CID: %s" %self._recvbuffer)
def __init__(
self,
chain: BlockChainService,
default_registry: Registry,
default_secret_registry: SecretRegistry,
private_key_bin,
transport,
config,
discovery=None,
):
if not isinstance(private_key_bin,
bytes) or len(private_key_bin) != 32:
raise ValueError('invalid private_key')
invalid_timeout = (
config['settle_timeout'] < NETTINGCHANNEL_SETTLE_TIMEOUT_MIN
or config['settle_timeout'] > NETTINGCHANNEL_SETTLE_TIMEOUT_MAX)
if invalid_timeout:
raise ValueError('settle_timeout must be in range [{}, {}]'.format(
NETTINGCHANNEL_SETTLE_TIMEOUT_MIN,
NETTINGCHANNEL_SETTLE_TIMEOUT_MAX,
))
self.tokens_to_connectionmanagers = dict()
self.identifier_to_results = defaultdict(list)
self.chain: BlockChainService = chain
self.default_registry = default_registry
self.default_secret_registry = default_secret_registry
self.config = config
self.privkey = private_key_bin
self.address = privatekey_to_address(private_key_bin)
if config['transport_type'] == 'udp':
endpoint_registration_event = gevent.spawn(
discovery.register,
self.address,
config['external_ip'],
config['external_port'],
)
endpoint_registration_event.link_exception(
endpoint_registry_exception_handler)
self.private_key = PrivateKey(private_key_bin)
self.pubkey = self.private_key.public_key.format(compressed=False)
self.transport = transport
self.blockchain_events = BlockchainEvents()
self.alarm = AlarmTask(chain)
self.shutdown_timeout = config['shutdown_timeout']
self.stop_event = Event()
self.start_event = Event()
self.chain.client.inject_stop_event(self.stop_event)
self.wal = None
self.database_path = config['database_path']
if self.database_path != ':memory:':
database_dir = os.path.dirname(config['database_path'])
os.makedirs(database_dir, exist_ok=True)
self.database_dir = database_dir
# Prevent concurrent acces to the same db
self.lock_file = os.path.join(self.database_dir, '.lock')
self.db_lock = filelock.FileLock(self.lock_file)
else:
self.database_path = ':memory:'
self.database_dir = None
self.lock_file = None
self.serialization_file = None
self.db_lock = None
if config['transport_type'] == 'udp':
# If the endpoint registration fails the node will quit, this must
# finish before starting the transport
endpoint_registration_event.join()
self.event_poll_lock = gevent.lock.Semaphore()
self.start()
def get(self, key):
event = Event()
item = {"operation": "get", "key": key, "event": event, "response": {}}
self.dispatcher.put(item)
return self._respond(item, event)
def __init__(self, acehost, aceAPIport, aceHTTPport, acehostslist, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = AceConfig.videoseekback
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._lock = threading.Condition(threading.Lock())
self._streamReaderConnection = None
self._streamReaderState = None
self._streamReaderQueue = deque()
self._engine_version_code = 0;
# Logger
logger = logging.getLogger('AceClientimport tracebacknt_init')
# Try to connect AceStream engine
try:
self._socket = telnetlib.Telnet(acehost, aceAPIport, connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = acehost, aceAPIport, aceHTTPport
logger.debug("Successfully connected to AceStream on %s:%d" % (acehost, aceAPIport))
except:
for AceEngine in acehostslist:
try:
self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1], connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[0], AceEngine[1], AceEngine[2]
logger.debug("Successfully connected to AceStream on %s:%d" % (AceEngine[0], AceEngine[1]))
break
except:
logger.debug("The are no alive AceStream on %s:%d" % (AceEngine[0], AceEngine[1]))
pass
# Spawning recvData greenlet
gevent.spawn(self._recvData)
gevent.sleep()
class MeekSession(RelaySession):
conn_pool = HTTPClientPool()
def __init__(self, socksconn, meek, timeout):
super(MeekSession, self).__init__(socksconn)
self.sessionid = session_id()
self.meek = meek
self.meektimeout = timeout
self.relay = self.meek.select_relay()
self.ca_certs = self.meek.ca_certs
self.httpclient = self.conn_pool.get(self.relay, self.ca_certs,
self.meektimeout)
self.udpsock = None
self.allsocks = [self.socksconn]
self.l2m_queue = Queue()
self.m2l_queue = Queue()
self.m_notifier = Event()
self.l_notifier = Event()
self.finish = Event()
self.m_notifier.clear()
self.l_notifier.clear()
self.finish.clear()
self.timer = SharedTimer(self.meektimeout)
def _stream_response(self, response):
try:
chunk = response.read(MAX_PAYLOAD_LENGTH)
while chunk:
log.debug("%s streaming DOWN %d bytes" %
(self.sessionid, len(chunk)))
yield chunk, ""
chunk = response.read(MAX_PAYLOAD_LENGTH)
except GeneratorExit:
response.release()
raise StopIteration
def meek_response(self, response, stream):
if stream:
return self._stream_response(response)
data = response.read()
response.release()
if not data:
return [("", "")]
if not self.udpsock:
return [(data, "")]
# parse UDP packets
log.debug("%s DOWN %d bytes" % (self.sessionid, len(data)))
lengths = get_meek_meta(response.headers, HEADER_UDP_PKTS).split(",")
pos = 0
pkts = []
for length in lengths:
nxt = pos + int(length)
pkts.append((data[pos:nxt], ""))
pos = nxt
return pkts
def meek_roundtrip(self, pkts):
headers = {
HEADER_SESSION_ID: self.sessionid,
HEADER_MSGTYPE: MSGTYPE_DATA,
'Host': self.relay.hostname,
'Content-Type': "application/octet-stream",
'Connection': "Keep-Alive",
}
stream = False
if not self.udpsock and "stream" in self.relay.properties:
stream = True
headers[HEADER_MODE] = MODE_STREAM
if pkts and self.udpsock:
lengths = str(",".join([str(len(p)) for p in pkts]))
headers[HEADER_UDP_PKTS] = lengths
data = "".join(pkts)
headers['Content-Length'] = str(len(data))
for _ in range(CLIENT_MAX_TRIES):
try:
log.debug("%s UP %d bytes" % (self.sessionid, len(data)))
resp = self.httpclient.post("/", body=data, headers=headers)
if resp.status_code != 200:
# meek server always give 200, so all non-200s mean external issues.
continue
err = get_meek_meta(resp.headers, HEADER_ERROR)
if err:
return [("", err)]
else:
try:
return self.meek_response(resp, stream)
except Exception as ex:
log.error(
"[Exception][meek_roundtrip - meek_response]: %s" %
str(ex))
resp.release()
return [("", "Data Format Error")]
except socket.timeout: # @UndefinedVariable
return [("", "timeout")]
except Exception as ex:
log.error("[Exception][meek_roundtrip]: %s" % str(ex))
gevent.sleep(CLIENT_RETRY_DELAY)
self.relay.failure += 1
return [("", "Max Retry (%d) Exceeded" % CLIENT_MAX_TRIES)]
def meek_sendrecv(self):
pkts = []
datalen = 0
while not self.l2m_queue.empty():
pkt = self.l2m_queue.get()
pkts.append(pkt)
datalen += len(pkt)
if datalen >= MAX_PAYLOAD_LENGTH:
for (resp, err) in self.meek_roundtrip(pkts):
yield (resp, err)
if err or not resp:
return
pkts = []
datalen = 0
for (resp, err) in self.meek_roundtrip(pkts):
yield (resp, err)
if err or not resp:
return
def meek_relay(self):
for (resp, err) in self.meek_sendrecv():
if err:
return err
if resp:
self.m2l_queue.put(resp)
self.l_notifier.set()
return ""
def meek_relay_thread(self):
interval = CLIENT_INITIAL_POLL_INTERVAL
while not self.finish.is_set():
try:
hasdata = self.m_notifier.wait(timeout=interval)
self.m_notifier.clear()
err = self.meek_relay()
if err:
break
if not hasdata:
interval *= CLIENT_POLL_INTERVAL_MULTIPLIER
if interval > CLIENT_MAX_POLL_INTERVAL:
interval = CLIENT_MAX_POLL_INTERVAL
except Exception as ex:
log.error("[Exception][meek_relay_thread]: %s" % str(ex))
break
self.finish.set()
def write_to_client(self, data):
if self.udpsock:
self.udpsock.sendto(data, self.last_clientaddr)
else:
self.socksconn.sendall(data)
def meek_write_to_client_thread(self):
while not self.finish.is_set():
try:
hasdata = self.l_notifier.wait(
timeout=CLIENT_MAX_POLL_INTERVAL)
self.l_notifier.clear()
if not hasdata:
self.timer.count(CLIENT_MAX_POLL_INTERVAL)
if self.timer.timeout():
break
else:
self.timer.reset()
while not self.m2l_queue.empty():
data = self.m2l_queue.get()
if data:
self.write_to_client(data)
except Exception as ex:
log.error("[Exception][meek_write_to_client_thread]: %s" %
str(ex))
break
self.finish.set()
def read_from_client(self, timeout):
readable, _, _ = select.select(self.allsocks, [], [],
CLIENT_MAX_POLL_INTERVAL)
if not readable:
return None
if self.socksconn in readable:
if self.udpsock:
raise RelaySessionError(
"unexcepted read-event from tcp socket in UDP session")
data = self.socksconn.recv(MAX_PAYLOAD_LENGTH)
if not data:
raise RelaySessionError("peer closed")
return data
if self.udpsock and self.udpsock in readable:
data, addr = self.udpsock.recvfrom(MAX_PAYLOAD_LENGTH)
if not self.valid_udp_client(addr):
return None
else:
self.last_clientaddr = addr
return data
def meek_read_from_client_thread(self):
while not self.finish.is_set():
try:
data = self.read_from_client(CLIENT_MAX_POLL_INTERVAL)
if not data:
self.timer.count(CLIENT_MAX_POLL_INTERVAL)
if self.timer.timeout():
break
else:
self.timer.reset()
self.l2m_queue.put(data)
self.m_notifier.set()
except Exception as ex:
log.error("[Exception][meek_read_from_client_thread]: %s" %
str(ex))
break
self.finish.set()
def proc_tcp_request(self, req):
self.l2m_queue.put(req.pack())
def relay_tcp(self):
read_thread = gevent.spawn(self.meek_read_from_client_thread)
write_thread = gevent.spawn(self.meek_write_to_client_thread)
relay_thread = gevent.spawn(self.meek_relay_thread)
# notify relay to send request
self.m_notifier.set()
[t.join() for t in (read_thread, write_thread, relay_thread)]
log.info("Session %s Ended" % self.sessionid)
def valid_udp_client(self, addr):
if self.client_associate[0] == "0.0.0.0" or \
self.client_associate[0] == "::":
return True
if self.client_associate == addr:
return True
return False
def cmd_udp_associate(self, req):
self.client_associate = (req.dstaddr, req.dstport)
self.last_clientaddr = self.client_associate
for (resp, err) in self.meek_roundtrip([req.pack()]):
if err:
return
if resp:
Reply(resp)
self.udpsock = bind_local_udp(self.socksconn)
if not self.udpsock:
request_fail(self.socksconn, req, GENERAL_SOCKS_SERVER_FAILURE)
return
self.track_sock(self.udpsock)
read_thread = gevent.spawn(self.meek_read_from_client_thread)
write_thread = gevent.spawn(self.meek_write_to_client_thread)
relay_thread = gevent.spawn(self.meek_relay_thread)
request_success(self.socksconn, *sock_addr_info(self.udpsock))
[t.join() for t in (read_thread, write_thread, relay_thread)]
log.info("Session %s Ended" % self.sessionid)
def meek_terminate(self):
headers = {
HEADER_SESSION_ID: self.sessionid,
HEADER_MSGTYPE: MSGTYPE_TERMINATE,
#'Content-Type': "application/octet-stream",
'Content-Length': "0",
'Connection': "Keep-Alive",
'Host': self.relay.hostname,
}
try:
self.httpclient.post("/", data="", headers=headers)
except:
pass
def clean(self):
self.meek_terminate()
for sock in self.allsocks:
sock.close()
#self.httpclient.close()
self.conn_pool.release(self.relay, self.httpclient)
def _run_app(self):
from raiden.ui.console import Console
from raiden.api.python import RaidenAPI
# this catches exceptions raised when waiting for the stalecheck to complete
try:
app_ = run_app(**self._options)
except (EthNodeCommunicationError, RequestsConnectionError):
print(ETHEREUM_NODE_COMMUNICATION_ERROR)
sys.exit(1)
domain_list = []
if self._options['rpccorsdomain']:
if ',' in self._options['rpccorsdomain']:
for domain in self._options['rpccorsdomain'].split(','):
domain_list.append(str(domain))
else:
domain_list.append(str(self._options['rpccorsdomain']))
self._raiden_api = RaidenAPI(app_.raiden)
api_server = None
if self._options['rpc']:
rest_api = RestAPI(self._raiden_api)
api_server = APIServer(
rest_api,
cors_domain_list=domain_list,
web_ui=self._options['web_ui'],
eth_rpc_endpoint=self._options['eth_rpc_endpoint'],
)
(api_host, api_port) = split_endpoint(self._options['api_address'])
try:
api_server.start(api_host, api_port)
except APIServerPortInUseError:
print(
'ERROR: API Address %s:%s is in use. '
'Use --api-address <host:port> to specify port to listen on.'
% (api_host, api_port), )
sys.exit(1)
print(
'The Raiden API RPC server is now running at http://{}:{}/.\n\n'
'See the Raiden documentation for all available endpoints at\n'
'http://raiden-network.readthedocs.io/en/stable/rest_api.html'.
format(
api_host,
api_port,
), )
if self._options['console']:
console = Console(app_)
console.start()
# spawn a greenlet to handle the version checking
gevent.spawn(check_version)
# spawn a greenlet to handle the gas reserve check
gevent.spawn(check_gas_reserve, app_.raiden)
self._startup_hook()
# wait for interrupt
event = Event()
gevent.signal(signal.SIGQUIT, event.set)
gevent.signal(signal.SIGTERM, event.set)
gevent.signal(signal.SIGINT, event.set)
try:
event.wait()
print('Signal received. Shutting down ...')
except (EthNodeCommunicationError, RequestsConnectionError):
print(ETHEREUM_NODE_COMMUNICATION_ERROR)
sys.exit(1)
except RaidenError as ex:
click.secho(f'FATAL: {ex}', fg='red')
except Exception as ex:
with NamedTemporaryFile(
'w',
prefix=
f'raiden-exception-{datetime.utcnow():%Y-%m-%dT%H-%M}',
suffix='.txt',
delete=False,
) as traceback_file:
traceback.print_exc(file=traceback_file)
click.secho(
f'FATAL: An unexpected exception occured. '
f'A traceback has been written to {traceback_file.name}\n'
f'{ex}',
fg='red',
)
finally:
self._shutdown_hook()
if api_server:
api_server.stop()
return app_
class MDSThrasher(Greenlet, Thrasher, object):
"""
MDSThrasher::
The MDSThrasher thrashes MDSs during execution of other tasks (workunits, etc).
The config is optional. Many of the config parameters are a a maximum value
to use when selecting a random value from a range. To always use the maximum
value, set no_random to true. The config is a dict containing some or all of:
max_thrash: [default: 1] the maximum number of active MDSs per FS that will be thrashed at
any given time.
max_thrash_delay: [default: 30] maximum number of seconds to delay before
thrashing again.
max_replay_thrash_delay: [default: 4] maximum number of seconds to delay while in
the replay state before thrashing.
max_revive_delay: [default: 10] maximum number of seconds to delay before
bringing back a thrashed MDS.
randomize: [default: true] enables randomization and use the max/min values
seed: [no default] seed the random number generator
thrash_in_replay: [default: 0.0] likelihood that the MDS will be thrashed
during replay. Value should be between 0.0 and 1.0.
thrash_max_mds: [default: 0.05] likelihood that the max_mds of the mds
cluster will be modified to a value [1, current) or (current, starting
max_mds]. Value should be between 0.0 and 1.0.
thrash_while_stopping: [default: false] thrash an MDS while there
are MDS in up:stopping (because max_mds was changed and some
MDS were deactivated).
thrash_weights: allows specific MDSs to be thrashed more/less frequently.
This option overrides anything specified by max_thrash. This option is a
dict containing mds.x: weight pairs. For example, [mds.a: 0.7, mds.b:
0.3, mds.c: 0.0]. Each weight is a value from 0.0 to 1.0. Any MDSs not
specified will be automatically given a weight of 0.0 (not thrashed).
For a given MDS, by default the trasher delays for up to
max_thrash_delay, trashes, waits for the MDS to recover, and iterates.
If a non-zero weight is specified for an MDS, for each iteration the
thrasher chooses whether to thrash during that iteration based on a
random value [0-1] not exceeding the weight of that MDS.
Examples::
The following example sets the likelihood that mds.a will be thrashed
to 80%, mds.b to 20%, and other MDSs will not be thrashed. It also sets the
likelihood that an MDS will be thrashed in replay to 40%.
Thrash weights do not have to sum to 1.
tasks:
- ceph:
- mds_thrash:
thrash_weights:
- mds.a: 0.8
- mds.b: 0.2
thrash_in_replay: 0.4
- ceph-fuse:
- workunit:
clients:
all: [suites/fsx.sh]
The following example disables randomization, and uses the max delay values:
tasks:
- ceph:
- mds_thrash:
max_thrash_delay: 10
max_revive_delay: 1
max_replay_thrash_delay: 4
"""
def __init__(self, ctx, manager, config, fs, max_mds):
super(MDSThrasher, self).__init__()
self.config = config
self.ctx = ctx
self.logger = log.getChild('fs.[{f}]'.format(f=fs.name))
self.fs = fs
self.manager = manager
self.max_mds = max_mds
self.name = 'thrasher.fs.[{f}]'.format(f=fs.name)
self.stopping = Event()
self.randomize = bool(self.config.get('randomize', True))
self.thrash_max_mds = float(self.config.get('thrash_max_mds', 0.05))
self.max_thrash = int(self.config.get('max_thrash', 1))
self.max_thrash_delay = float(self.config.get('thrash_delay', 120.0))
self.thrash_in_replay = float(
self.config.get('thrash_in_replay', False))
assert self.thrash_in_replay >= 0.0 and self.thrash_in_replay <= 1.0, 'thrash_in_replay ({v}) must be between [0.0, 1.0]'.format(
v=self.thrash_in_replay)
self.max_replay_thrash_delay = float(
self.config.get('max_replay_thrash_delay', 4.0))
self.max_revive_delay = float(self.config.get('max_revive_delay',
10.0))
def _run(self):
try:
self.do_thrash()
except Exception as e:
# Log exceptions here so we get the full backtrace (gevent loses them).
# Also allow successful completion as gevent exception handling is a broken mess:
#
# 2017-02-03T14:34:01.259 CRITICAL:root: File "gevent.libev.corecext.pyx", line 367, in gevent.libev.corecext.loop.handle_error (src/gevent/libev/gevent.corecext.c:5051)
# File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/gevent/hub.py", line 558, in handle_error
# self.print_exception(context, type, value, tb)
# File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/gevent/hub.py", line 605, in print_exception
# traceback.print_exception(type, value, tb, file=errstream)
# File "/usr/lib/python2.7/traceback.py", line 124, in print_exception
# _print(file, 'Traceback (most recent call last):')
# File "/usr/lib/python2.7/traceback.py", line 13, in _print
# file.write(str+terminator)
# 2017-02-03T14:34:01.261 CRITICAL:root:IOError
self.exception = e
self.logger.exception("exception:")
# allow successful completion so gevent doesn't see an exception...
def log(self, x):
"""Write data to the logger assigned to MDSThrasher"""
self.logger.info(x)
def stop(self):
self.stopping.set()
def kill_mds(self, mds):
if self.config.get('powercycle'):
(remote, ) = (self.ctx.cluster.only(
'mds.{m}'.format(m=mds)).remotes.iterkeys())
self.log('kill_mds on mds.{m} doing powercycle of {s}'.format(
m=mds, s=remote.name))
self._assert_ipmi(remote)
remote.console.power_off()
else:
self.ctx.daemons.get_daemon('mds', mds).stop()
@staticmethod
def _assert_ipmi(remote):
assert remote.console.has_ipmi_credentials, (
"powercycling requested but RemoteConsole is not "
"initialized. Check ipmi config.")
def revive_mds(self, mds):
"""
Revive mds -- do an ipmpi powercycle (if indicated by the config)
and then restart.
"""
if self.config.get('powercycle'):
(remote, ) = (self.ctx.cluster.only(
'mds.{m}'.format(m=mds)).remotes.iterkeys())
self.log('revive_mds on mds.{m} doing powercycle of {s}'.format(
m=mds, s=remote.name))
self._assert_ipmi(remote)
remote.console.power_on()
self.manager.make_admin_daemon_dir(self.ctx, remote)
args = []
self.ctx.daemons.get_daemon('mds', mds).restart(*args)
def wait_for_stable(self, rank=None, gid=None):
self.log('waiting for mds cluster to stabilize...')
for itercount in itertools.count():
status = self.fs.status()
max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds']
ranks = list(status.get_ranks(self.fs.id))
stopping = filter(lambda info: "up:stopping" == info['state'],
ranks)
actives = filter(
lambda info: "up:active" == info['state'] and "laggy_since"
not in info, ranks)
if not bool(self.config.get('thrash_while_stopping',
False)) and len(stopping) > 0:
if itercount % 5 == 0:
self.log(
'cluster is considered unstable while MDS are in up:stopping (!thrash_while_stopping)'
)
else:
if rank is not None:
try:
info = status.get_rank(self.fs.id, rank)
if info['gid'] != gid and "up:active" == info['state']:
self.log(
'mds.{name} has gained rank={rank}, replacing gid={gid}'
.format(name=info['name'], rank=rank, gid=gid))
return status
except:
pass # no rank present
if len(actives) >= max_mds:
# no replacement can occur!
self.log(
"cluster has %d actives (max_mds is %d), no MDS can replace rank %d"
.format(len(actives), max_mds, rank))
return status
else:
if len(actives) == max_mds:
self.log(
'mds cluster has {count} alive and active, now stable!'
.format(count=len(actives)))
return status, None
if itercount > 300 / 2: # 5 minutes
raise RuntimeError('timeout waiting for cluster to stabilize')
elif itercount % 5 == 0:
self.log('mds map: {status}'.format(status=status))
else:
self.log('no change')
sleep(2)
def do_thrash(self):
"""
Perform the random thrashing action
"""
self.log('starting mds_do_thrash for fs {fs}'.format(fs=self.fs.name))
stats = {
"max_mds": 0,
"deactivate": 0,
"kill": 0,
}
while not self.stopping.is_set():
delay = self.max_thrash_delay
if self.randomize:
delay = random.randrange(0.0, self.max_thrash_delay)
if delay > 0.0:
self.log('waiting for {delay} secs before thrashing'.format(
delay=delay))
self.stopping.wait(delay)
if self.stopping.is_set():
continue
status = self.fs.status()
if random.random() <= self.thrash_max_mds:
max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds']
options = range(1, max_mds) + range(max_mds + 1,
self.max_mds + 1)
if len(options) > 0:
sample = random.sample(options, 1)
new_max_mds = sample[0]
self.log('thrashing max_mds: %d -> %d' %
(max_mds, new_max_mds))
self.fs.set_max_mds(new_max_mds)
stats['max_mds'] += 1
self.wait_for_stable()
count = 0
for info in status.get_ranks(self.fs.id):
name = info['name']
label = 'mds.' + name
rank = info['rank']
gid = info['gid']
# if thrash_weights isn't specified and we've reached max_thrash,
# we're done
count = count + 1
if 'thrash_weights' not in self.config and count > self.max_thrash:
break
weight = 1.0
if 'thrash_weights' in self.config:
weight = self.config['thrash_weights'].get(label, '0.0')
skip = random.randrange(0.0, 1.0)
if weight <= skip:
self.log(
'skipping thrash iteration with skip ({skip}) > weight ({weight})'
.format(skip=skip, weight=weight))
continue
self.log('kill {label} (rank={rank})'.format(label=label,
rank=rank))
self.kill_mds(name)
stats['kill'] += 1
# wait for mon to report killed mds as crashed
last_laggy_since = None
itercount = 0
while True:
status = self.fs.status()
info = status.get_mds(name)
if not info:
break
if 'laggy_since' in info:
last_laggy_since = info['laggy_since']
break
if any([(f == name) for f in status.get_fsmap(self.fs.id)
['mdsmap']['failed']]):
break
self.log(
'waiting till mds map indicates {label} is laggy/crashed, in failed state, or {label} is removed from mdsmap'
.format(label=label))
itercount = itercount + 1
if itercount > 10:
self.log('mds map: {status}'.format(status=status))
sleep(2)
if last_laggy_since:
self.log(
'{label} reported laggy/crashed since: {since}'.format(
label=label, since=last_laggy_since))
else:
self.log('{label} down, removed from mdsmap'.format(
label=label, since=last_laggy_since))
# wait for a standby mds to takeover and become active
status = self.wait_for_stable(rank, gid)
# wait for a while before restarting old active to become new
# standby
delay = self.max_revive_delay
if self.randomize:
delay = random.randrange(0.0, self.max_revive_delay)
self.log(
'waiting for {delay} secs before reviving {label}'.format(
delay=delay, label=label))
sleep(delay)
self.log('reviving {label}'.format(label=label))
self.revive_mds(name)
for itercount in itertools.count():
if itercount > 300 / 2: # 5 minutes
raise RuntimeError('timeout waiting for MDS to revive')
status = self.fs.status()
info = status.get_mds(name)
if info and info['state'] in ('up:standby',
'up:standby-replay',
'up:active'):
self.log('{label} reported in {state} state'.format(
label=label, state=info['state']))
break
self.log(
'waiting till mds map indicates {label} is in active, standby or standby-replay'
.format(label=label))
sleep(2)
for stat in stats:
self.log("stat['{key}'] = {value}".format(key=stat,
value=stats[stat]))
class CommandDispatcher(object):
#this class contains a queue where request
def __init__(self, vbaware, verbose=False):
#have a queue , in case of not my vbucket error
#let's reinitialize the config/memcached socket connections ?
self.queue = Queue(10000)
self.status = "initialized"
self.vbaware = vbaware
self.reconfig_callback = self.vbaware.reconfig_vbucket_map
self.start_connection_callback = self.vbaware.start_vbucket_connection
self.restart_connection_callback = self.vbaware.restart_vbucket_connection
self.verbose = verbose
self.log = logger.logger("CommandDispatcher")
self._dispatcher_stopped_event = Event()
def put(self, item):
try:
self.queue.put(item, False)
except Full:
#TODO: add a better error message here
raise Exception("queue is full")
def shutdown(self):
if self.status != "shutdown":
self.status = "shutdown"
if self.verbose:
self.log.info("dispatcher shutdown command received")
self._dispatcher_stopped_event.wait(2)
def reconfig_completed(self):
self.status = "ok"
def dispatch(self):
while self.status != "shutdown" or (self.status == "shutdown"
and self.queue.qsize() > 0):
#wait if its reconfiguring the vbucket-map
if self.status == "vbucketmap-configuration":
continue
try:
item = self.queue.get(block=True, timeout=1)
if item:
try:
self.do(item)
# do will only raise not_my_vbucket_exception,
# EOF and socket.error
except MemcachedError as ex:
# if we get a not_my_vbucket then requeue item
# with fast forward map vbucket
self.log.error(ex)
self.reconfig_callback(ex.vbucket)
self.start_connection_callback(ex.vbucket)
item["fastforward"] = True
self.queue.put(item)
except exceptions.EOFError as ex:
# we go an EOF error, restart the connection
self.log.error(ex)
self.restart_connection_callback(ex.vbucket)
self.queue.put(item)
except socket.error as ex:
# we got a socket error, restart the connection
self.log.error(ex)
self.restart_connection_callback(ex.vbucket)
self.queue.put(item)
except Empty:
pass
if self.verbose:
self.log.info("dispatcher stopped")
self._dispatcher_stopped_event.set()
def _raise_if_recoverable(self, ex, item):
if isinstance(ex, MemcachedError) and ex.status == 7:
ex.vbucket = item["vbucket"]
print ex
self.log.error(
"got not my vb error. key: {0}, vbucket: {1}".format(
item["key"], item["vbucket"]))
raise ex
if isinstance(ex, exceptions.EOFError):
ex.vbucket = item["vbucket"]
print ex
self.log.error("got EOF")
raise ex
if isinstance(ex, socket.error):
ex.vbucket = item["vbucket"]
print ex
self.log.error("got socket.error")
raise ex
item["response"]["error"] = ex
def do(self, item):
#find which vbucket this belongs to and then run the operation on that ?
if "key" in item:
item["vbucket"] = self.vbaware.vbucketid(item["key"])
if not "fastforward" in item:
item["fastforward"] = False
item["response"]["return"] = None
if item["operation"] == "get":
key = item["key"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).get(key)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "set":
key = item["key"]
expiry = item["expiry"]
flags = item["flags"]
value = item["value"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).set(key, expiry, flags, value)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "add":
key = item["key"]
expiry = item["expiry"]
flags = item["flags"]
value = item["value"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).add(key, expiry, flags, value)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "replace":
key = item["key"]
expiry = item["expiry"]
flags = item["flags"]
value = item["value"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).replace(key, expiry, flags,
value)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "delete":
key = item["key"]
cas = item["cas"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).delete(key, cas)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "prepend":
key = item["key"]
cas = item["cas"]
value = item["value"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).prepend(key, value, cas)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "append":
key = item["key"]
cas = item["cas"]
value = item["value"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).append(key, value, cas)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "getl":
key = item["key"]
expiry = item["expiry"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).getl(key, expiry)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "gat":
key = item["key"]
expiry = item["expiry"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).gat(key, expiry)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "touch":
key = item["key"]
expiry = item["expiry"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).touch(key, expiry)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "incr":
key = item["key"]
amount = item["amount"]
init = item["init"]
expiry = item["expiry"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).incr(key, amount, init, expiry)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "decr":
key = item["key"]
amount = item["amount"]
init = item["init"]
expiry = item["expiry"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).decr(key, amount, init, expiry)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
elif item["operation"] == "cas":
key = item["key"]
expiry = item["expiry"]
flags = item["flags"]
old_value = item["old_value"]
value = item["value"]
try:
item["response"]["return"] = self.vbaware.memcached(
key, item["fastforward"]).cas(key, expiry, flags,
old_value, value)
except Exception as ex:
self._raise_if_recoverable(ex, item)
item["event"].set()
def __init__(self, web3, mine_sleep=1):
super().__init__()
self.web3 = web3
self.mine_sleep = mine_sleep
self.stop = Event()
# pylint: disable=E0401
from shutil import copyfile
from gevent.event import Event
import sys
sys.path.append('./python')
from config import config
config.videoPreview = ''
config.progressDetail = -1
from worker import context
from video import SR_vid
context.stopFlag = Event()
context.shared = None
video = 'upload\\realshort.mp4'
copyfile('test\\realshort.mp4', video)
steps = [{'op': 'decode'}, {'op': 'range'}, {'codec': 'h264_nvenc -pix_fmt yuv420p', 'op': 'encode', 'file': 'download/realshort.mkv'}]
print(SR_vid(video, False, *steps))
