class Worker(object): ''' 子进程运行的代码,通过起一个协程来和主进程通信 包括接受任务分配请求,退出信号(零字节包),及反馈任务执行进度 然后主协程等待停止信号并中止进程(stop_event用于协程间同步)。 ''' def __init__(self, address): self.address = address self.stop_event = Event() self.wp = open('fetched_%s.txt' % os.getpid(), 'w') self.uf_url_p = open('unfetched_%s.txt' % os.getpid(), 'w') gevent.spawn(self.communicate) self.stop_event.wait() print 'worker(%s):will stop' % os.getpid() def exec_task(self, url): try: article = g.extract(url=url) title = article.title content = article.cleaned_text if content != None and len(content) > 0: self.wp.write( url ) #self.wp.write( ' ' ) #self.wp.write( title.replace(' ', '').replace('\n', '').encode('utf8') ) self.wp.write( ' ' ) self.wp.write( content.replace('\n', '').encode('utf8') ) self.wp.write( '\n' ) else: self.uf_url_p.write( url ) self.uf_url_p.write( '\n' ) except (TypeError, IncompleteRead, AttributeError), e: return
class ConditionPoller(Thread): """ generic polling mechanism: every interval seconds, check if condition returns a true value. if so, pass the value to callback if condition or callback raise exception, stop polling. """ def __init__(self, condition, condition_callback, exception_callback, interval): self.polling_interval = interval self._shutdown_now = Event() self._condition = condition self._callback = condition_callback self._on_exception = exception_callback super(ConditionPoller,self).__init__() def shutdown(self): self.is_shutting_down = True self._shutdown_now.set() def run(self): try: while not self._shutdown_now.is_set(): self._check_condition() self._shutdown_now.wait(self.polling_interval) except: log.error('thread failed', exc_info=True) def _check_condition(self): try: value = self._condition() if value: self._callback(value) except Exception as e: log.debug('stopping poller after exception', exc_info=True) self.shutdown() if self._on_exception: self._on_exception(e) def start(self): super(ConditionPoller,self).start()
def waitForPublishResult(self, eventName): try: event = self.eventPublishResult[eventName] except KeyError: event = Event() self.eventPublishResult[eventName] = event event.wait()
def StartGRpcServer(sJsonRpcHostPort, sModule, cnMaster, cnWorker): from gevent.event import Event from grpc import RpcServer lsHP = sJsonRpcHostPort.split(':') if len(lsHP)<2: PrintTimeMsg("sJsonRpcHostPort=(%s) error!" % sJsonRpcHostPort) return endpoint = (lsHP[0], int(lsHP[1])) try:#WeiYF.20150414 经测试,采用importModuleClass才行 #classMaster = eval("CMaster") #eval(cnMaster) #classWorker = eval(cnWorker) # classMaster = importClassString("grcs.grsVtrbtctx.CMaster") # classWorker = importClassString("grcs.grsVtrbtctx.CWorker") classMaster = importModuleClass(sModule,cnMaster) classWorker = importModuleClass(sModule,cnWorker) master = classMaster(endpoint) svr = RpcServer() svr.bind(endpoint) t1 = classWorker() svr.register(t1) svr.register(master) t1.set_master(svr.get_export_proxy(master)) svr.start() PrintTimeMsg('Starting GRPC server(%s,%s)...' % (sModule,sJsonRpcHostPort)) wait = Event() wait.wait() #except KeyboardInterrupt: except Exception, e: import traceback traceback.print_exc() #WeiYF.20151022 打印异常整个堆栈 这个对于动态加载非常有用 PrintTimeMsg('StartGRpcServer.Exception.e=(%s)' % (str(e)))
class TestEvent(Actor): '''**Generates a test event at the chosen interval.** This module is only available for testing purposes and has further hardly any use. Events have following format: { "header":{}, "data":"test" } Parameters: - name (str): The instance name when initiated. - interval (float): The interval in seconds between each generated event. Should have a value > 0. default: 1 Queues: - outbox: Contains the generated events. ''' def __init__(self, name, interval=1): Actor.__init__(self, name, setupbasic=False) self.createQueue("outbox") self.name = name self.interval=interval if interval == 0: self.sleep = self.doNoSleep else: self.sleep = self.doSleep self.throttle=Event() self.throttle.set() def preHook(self): spawn(self.go) def go(self): switcher = self.getContextSwitcher(100) while switcher(): self.throttle.wait() try: self.queuepool.outbox.put({"header":{},"data":"test"}) except (QueueFull, QueueLocked): self.queuepool.outbox.waitUntilPutAllowed() self.sleep(self.interval) def doSleep(self, interval): sleep(interval) def doNoSleep(self, interval): pass def enableThrottling(self): self.throttle.clear() def disableThrottling(self): self.throttle.set()
def test_apply_async(self): done = Event() def some_work(x): done.set() pool = self.klass(2) pool.apply_async(some_work, ('x', )) done.wait()
def run(ctx, dev): """Start the client""" # create app app = EthApp(ctx.obj['config']) if dev: gevent.get_hub().SYSTEM_ERROR = BaseException try: ctx.obj['config']['client_version'] += '/' + os.getlogin() except: log.warn("can't get and add login name to client_version") pass # register services for service in services: assert issubclass(service, BaseService) if service.name not in app.config['deactivated_services']: assert service.name not in app.services service.register_with_app(app) assert hasattr(app.services, service.name) # start app app.start() # wait for interupt evt = Event() gevent.signal(signal.SIGQUIT, evt.set) gevent.signal(signal.SIGTERM, evt.set) gevent.signal(signal.SIGINT, evt.set) evt.wait() # finally stop app.stop()
class ClusterCoordinator(Service): port = Setting('cluster_port', default=4440) def __init__(self, identity, leader=None, cluster=None): leader = leader or identity self.server = PeerServer(self, identity) self.client = PeerClient(self, leader, identity) self.set = cluster or ObservableSet() self.promoted = Event() self.add_service(self.server) if leader != identity: self.add_service(self.client) self.is_leader = False else: self.is_leader = True def wait_for_promotion(self): self.promoted.wait() @property def leader(self): return self.client.leader @property def identity(self): return self.client.identity
class Worker(object): ''' 子进程运行的代码,通过起一个协程来和主进程通信 包括接受任务分配请求,退出信号(零字节包),及反馈任务执行进度 然后主协程等待停止信号并中止进程(stop_event用于协程间同步)。 ''' def __init__(self, url): self.url = url self.stop_event = Event() gevent.spawn(self.communicate) self.stop_event.wait() print 'worker(%s):will stop' % os.getpid() def exec_task(self, task): print 'worker(%s):execute task:%s' % (os.getpid(), task.rstrip('\n')) def communicate(self): print 'worker(%s):started' % os.getpid() client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client.connect(self.url) fp = client.makefile() while True: line = fp.readline() if not line: self.stop_event.set() break '单独起一个协程去执行任务,防止通信协程阻塞' gevent.spawn(self.exec_task, line)
def test_client_shutdown(copper_client): have_stream = Event() may_respond = Event() def handler(stream): have_stream.set() may_respond.wait() stream.write('Hello, world!') with copper_client.publish('test:helloworld', handler): with copper_client.subscribe('test:helloworld') as sub: with sub.open() as stream: have_stream.wait() with pytest.raises(Timeout): with Timeout(0.005): # This initiates the shutdown, but it should not # complete on its own, because handler is still # running. Being stopped with a timeout does not # stop the shutdown procedure. copper_client.shutdown(unpublish=False) # Verify our handler can still reply successfully may_respond.set() assert stream.read() == 'Hello, world!' with sub.open() as stream: # Verify any new streams fail with ECONNSHUTDOWN (since our # code didn't unpublish the service), and don't reach our # handler. with pytest.raises(ConnectionShutdownError): stream.read() # Verify shutdown now finishes successfully. copper_client.shutdown()
def test_reentrant(self): pool = self.klass(1) result = pool.apply(pool.apply, (lambda a: a + 1, (5, ))) self.assertEqual(result, 6) evt = Event() pool.apply_async(evt.set) evt.wait()
def test_with_remoting(defer): random_data = str(random.randint(0, 10000000000)) node1 = Node('localhost:20001', enable_remoting=True) defer(node1.stop) node2 = Node('localhost:20002', enable_remoting=True) defer(node2.stop) f_src = tempfile.NamedTemporaryFile() f_src.write(random_data) f_src.flush() class Sender(Actor): def run(self, receiver): ref = serve_file(f_src.name) receiver << ref class Receiver(Actor): def receive(self, fref): for _ in range(2): fetched_path = fref.fetch() with open(fetched_path) as f_dst: eq_(random_data, f_dst.read()) received.set() received = Event() node2.spawn(Receiver, name='receiver') receiver_ref = node1.lookup_str('localhost:20002/receiver') node1.spawn(Sender.using(receiver=receiver_ref)) received.wait()
class RecurringTask(object): def __init__(self, interval, fn): self.interval = interval self.fn = fn self._wakeup = Event() self._stopped = Event() self._gthread = None def touch(self): """Make sure the task is executed now.""" self._wakeup.set() def start(self): self._gthread = gevent.spawn(self._run) def stop(self): self._stopped.set() self._wakeup.set() def _run(self): while not self._stopped.is_set(): self.fn() self._wakeup.wait(timeout=self.interval) self._wakeup.clear()
class Lock(object): def __init__(self, etcd, key, name, ttl=30): """.""" self.etcd = etcd self.key = key self.name = name self._gthread = None self._ttl = ttl self._stopped = Event() def _heartbeat(self): while True: self._stopped.wait(self._ttl / 2) if self._stopped.is_set(): break self.etcd.testandset(self.key, self.name, self.name, ttl=self._ttl) def lock(self): # This is to work around bugs in etcd. Not very atomic # at all :( while True: try: e = self.etcd.get(self.key) except EtcdError, err: logging.error("lock: %s: error: %r" % ( self.key, err)) self.etcd.set(self.key, self.name) self._gthread = gevent.spawn(self._heartbeat) break else: time.sleep(self._ttl / 2)
class _FormationCache(object): """A cache of instance data for a formation.""" def __init__(self, client, form_name, factory, interval): self.client = client self.form_name = form_name self.factory = factory self.interval = interval self._gthread = None self._cache = {} self._stopped = Event() self._running = Event() def start(self): self._gthread = gevent.spawn(self._loop) self._running.wait(timeout=0.1) return self def stop(self, timeout=None): self._stopped.set() self._gthread.join(timeout) def _update(self): self._cache = self.client.query_formation(self.form_name, self.factory) def _loop(self): while not self._stopped.isSet(): self._update() self._running.set() self._stopped.wait(self.interval) def query(self): """Return all instances and their names.""" return dict(self._cache)
class _Registration(object): """A service registration.""" def __init__(self, client, form_name, instance_name, data, interval=3): self.stopped = Event() self.client = client self.form_name = form_name self.instance_name = instance_name self.data = data self.interval = interval self.gthread = None def _loop(self): uri = '/%s/%s' % (self.form_name, self.instance_name) while not self.stopped.isSet(): response = self.client._request('PUT', uri, data=json.dumps(self.data)) self.stopped.wait(self.interval) def start(self): self.gthread = gevent.spawn(self._loop) return self def stop(self, timeout=None): self.stopped.set() self.gthread.join(timeout)
def test_semaphore(self): edge = APIEdge(MockApp(), self.get_settings()) api = edge.app.api edge.max_concurrent_calls = 1 in_first_method = Event() finish_first_method = Event() def first_method(): in_first_method.set() finish_first_method.wait() api.first_method = first_method in_second_method = Event() def second_method(): in_second_method.set() api.second_method = second_method gevent.spawn(edge.execute, Call("first_method")) in_first_method.wait() gevent.spawn(edge.execute, Call("second_method")) gevent.sleep(0) assert_logged("too many concurrent callers") assert not in_second_method.is_set() finish_first_method.set() in_second_method.wait() self.assert_edge_clean(edge)
class Pinger(Greenlet): """ Very simple test 'app' """ def __init__(self, id): super(Pinger,self).__init__() self.event = Event() self.conn = None self.id = id def _run(self): logger.debug("Pinger starting") self.conn = connection.AMQPConnection(self) self.conn.connect() #self.conn.connection.join() self.event.wait() logger.debug("Pinger exiting") self.amqp.close() self.conn.close() def on_connect(self, connection): self.amqp = PingerAMQPManager(connection, self, self.id) def handle_message(self, message): if message.routing_key.endswith('pinger.exit'): #self.conn.connection.close() self.event.set()
class Console(BaseService): """A service starting an interactive ipython session when receiving the SIGSTP signal (e.g. via keyboard shortcut CTRL-Z). """ name = 'console' def __init__(self, app): super(Console, self).__init__(app) self.interrupt = Event() gevent.signal(signal.SIGTSTP, self.interrupt.set) self.console_locals = [] def start(self): super(Console, self).start() self.console_locals = {} self.console_locals.update(self.app.services) self.console_locals['app'] = self.app def _run(self): while True: self.interrupt.wait() IPython.start_ipython(argv=['--gui', 'gevent'], user_ns=self.console_locals) self.interrupt.clear()
class Chat(object): def __init__(self): self.new_msg_event = Event() def write_message(self, request): if not request.user.is_authenticated() or request.method != 'POST': return HttpResponse(status=404) form = MessageForm(request.POST) output = dict(success=False) if form.is_valid(): form.save(request.user) output['success'] = True else: output['errors'] = form.get_errors() self.new_msg_event.set() self.new_msg_event.clear() return HttpResponse(json.dumps(output)) def get_messages(self, request): if not request.user.is_authenticated(): return HttpResponse(status=404) pk = int(request.GET.get('pk', 1)) messages = [{'created_at': DateFormat(el.created_at).format('H:i:s'), 'username': el.username, 'pk': el.pk, 'msg': el.msg} for el in Message.objects.filter(pk__gt=int(pk)) .order_by('-created_at')[:100]] if not messages: self.new_msg_event.wait() return HttpResponse(json.dumps(messages[::-1]))
class Chat(object): def __init__(self): # at some point, may want to implement a buffer for messages # self.buffer = [] self.msg_event = Event() def index(self, request): form = models.MsgForm() msg_list = models.Msg.objects.all() return render(request, 'index.html', { 'form': form, 'msg_list': msg_list, }) def send(self, request): if request.method == 'POST': form = models.MsgForm(request.POST) if form.is_valid(): form.save() # tell everyone who's waiting on msg_event that a msg was just # posted self.msg_event.set() self.msg_event.clear() return HttpResponse(json.dumps(True), mimetype='application/json') return HttpResponse(json.dumps(False), mimetype='application/json') def update(self, request): check_time = datetime.now() # wait for next msg post self.msg_event.wait() msg_list = models.Msg.objects.filter(time_stamp__gte=check_time) return HttpResponse(serializers.serialize('xml', msg_list), mimetype='text/xml')
class StdioPipedWebSocketClient(WebSocketClient): def __init__(self, scheme, host, port, path, opts): url = "{0}://{1}:{2}{3}".format(scheme, host, port, path) WebSocketClient.__init__(self, url) self.path = path self.shutdown_cond = Event() self.opts = opts self.iohelper = common.StdioPipedWebSocketHelper(self.shutdown_cond, opts) def received_message(self, m): #TODO here we can retrieve the msg self.iohelper.received_message(self, m) def opened(self): if self.opts.verbosity >= 1: peername, peerport = self.sock.getpeername() print >> sys.stderr, "[%s] %d open for path '%s'" % (peername, peerport, self.path) self.iohelper.opened(self) def closed(self, code, reason): self.shutdown_cond.set() def connect_and_wait(self): self.connect() self.shutdown_cond.wait()
class ServerRack(object): def __init__(self, servers): self.servers = servers self.ev = Event() def start(self): started = [] try: for server in self.servers[:]: server.start() started.append(server) name = getattr(server, 'name', None) or server.__class__.__name__ or 'Server' except: self.stop(started) raise def serve_forever(self): self.start() self.ev.wait() def stop(self, servers=None): if servers is None: servers = self.servers[:] for server in servers: try: server.stop() except: if hasattr(server, 'loop'): #gevent >= 1.0 server.loop.handle_error(server.stop, *sys.exc_info()) else: # gevent <= 0.13 import traceback traceback.print_exc() self.ev.set()
class GeventCursor(net.Cursor): def __init__(self, *args, **kwargs): super(GeventCursor, self).__init__(*args, **kwargs) self.new_response = Event() def __iter__(self): return self def __next__(self): return self._get_next(None) def _empty_error(self): return GeventCursorEmpty() def _extend(self, res): super(GeventCursor, self)._extend(res) self.new_response.set() self.new_response.clear() def _get_next(self, timeout): with gevent.Timeout(timeout, RqlTimeoutError()) as timeout: self._maybe_fetch_batch() while len(self.items) == 0: if self.error is not None: raise self.error self.new_response.wait() return self.items.popleft()
def evt_user_input(cls, arg): trans, ilet = arg evt = Event() ilet.event = evt process_msg(('evt_user_input', arg)) evt.wait() return ilet
def run(ctx, dev, nodial, fake): """Start the client ( --dev to stop on error)""" config = ctx.obj['config'] if nodial: # config['deactivated_services'].append(PeerManager.name) # config['deactivated_services'].append(NodeDiscovery.name) config['discovery']['bootstrap_nodes'] = [] config['discovery']['listen_port'] = 29873 config['p2p']['listen_port'] = 29873 config['p2p']['min_peers'] = 0 if fake: from ethereum import blocks blocks.GENESIS_DIFFICULTY = 1024 blocks.BLOCK_DIFF_FACTOR = 16 blocks.MIN_GAS_LIMIT = blocks.GENESIS_GAS_LIMIT / 2 # workaround for genesis.json hack blocks.GENESIS_JSON["difficulty"] = blocks.int_to_hex(blocks.GENESIS_DIFFICULTY) # create app app = EthApp(config) # development mode if dev: gevent.get_hub().SYSTEM_ERROR = BaseException try: config['client_version'] += '/' + os.getlogin() except: log.warn("can't get and add login name to client_version") pass # dump config konfig.dump_config(config) # register services for service in services: assert issubclass(service, BaseService) if service.name not in app.config['deactivated_services']: assert service.name not in app.services service.register_with_app(app) assert hasattr(app.services, service.name) unlock_accounts(ctx.obj['unlock'], app.services.accounts, password=ctx.obj['password']) # start app log.info('starting') app.start() if config['post_app_start_callback'] is not None: config['post_app_start_callback'](app) # wait for interrupt evt = Event() gevent.signal(signal.SIGQUIT, evt.set) gevent.signal(signal.SIGTERM, evt.set) gevent.signal(signal.SIGINT, evt.set) evt.wait() # finally stop app.stop()
class WSClientTransport(WebSocketClient): APP_FACTORY = None def __init__(self, url): self._close_event = Event() # patch socket.sendall to protect it with lock, # in order to prevent sending data from multiple greenlets concurrently WebSocketClient.__init__(self, url) self._app = None self._lock = RLock() _sendall = self.sock.sendall def sendall(data): self._lock.acquire() try: _sendall(data) except: raise finally: self._lock.release() self.sock.sendall = sendall def connect(self): super(WSClientTransport, self).connect() self._app = self.APP_FACTORY(self) log.info("Connected to websocket server {0}".format(self.url)) def closed(self, code, reason=None): app, self._app = self._app, None if app: app.on_close() self._close_event.set() def ponged(self, pong): pass def received_message(self, message): log.debug("Received message {0}".format(message)) if self._app: self._app.on_received_packet(STRING(message)) else: log.warning('Websocket client app already closed') def send_packet(self, data): log.debug("Sending message {0}".format(data)) self.send(data) def force_shutdown(self): # called by the upper layer, and no callback will be possible when closed self._app = None self.close() self._close_event.set() log.info('Websocket client closed') def wait_close(self): self._close_event.wait() def app(self): return self._app
class Master(): ''' 主进程运行代码,启动单独协程监听一个端口以供子进程连接和通信用, 通过subprocess.Popen启动CPU个数个子进程,注册SIGTERM信号以便在 KILL自己时通知子进程退出,主协程等待停止事件并退出主 ''' def __init__(self, address): self.address = address self.workers = [] self.stop_event = Event() gevent.spawn(self.communicate) gevent.sleep(0) #让communicate协程有机会执行,否则子进程会先启动 self.process = [subprocess.Popen(('python',sys.argv[0],'worker')) for i in xrange(12)] #启动multiprocessing.cpucount-1个子进程 gevent.signal(signal.SIGTERM, self.stop) #拦截kill信号 gevent.spawn(self.start) #分发任务 self.stop_event.wait() def communicate(self): print 'master(%s):started' % os.getpid() server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server.bind(address) server.listen(1024) while True: worker, addr = server.accept() print 'master(%s):new worker' % os.getpid() self.workers.append(worker) def stop(self): print 'master stop' for worker in self.workers: worker.close() for p in self.process: p.wait() self.stop_event.set() def start(self): while not self.workers: gevent.sleep(1) continue with open(url_file, 'rb') as fp: line_count = 0 t = time.time() for line in fp: url = line.split()[0] worker = random.choice(self.workers) worker.send(url + '\n') line_count += 1 if line_count % 1000 == 0: print 'process %s line used %s seconds' % (line_count, time.time() - t) gevent.sleep(10)
class Session: def __init__(self,handler,tg,chatid,user,name): self.handler=handler self.tg = tg self.name = name self.user=user self.chatid=chatid self.event=None self.returnmsg=None def _activate(self): self.handler.activeSessions[self.user]=self def _processmarkup(self,markup): if markup!=None: markup2={} markup2["resize_keyboard"]=True markup2["one_time_keyboard"]=True markup2["keyboard"]=markup return j.data.serializer.json.dumps(markup2) return markup def getArgument(self,key,descr="",global_=False,markup=None,presetvalue=None): if global_: hkey="config" else: hkey=self.name res=self.handler.redisconfig.hget(hkey,key) if res==None: # if descr=="": # session.send_message("Cannot find global variable: '%s' please define by using '%s=yourinput'."%(key,key)) # else: if presetvalue!=None: res=presetvalue elif descr=="": res=self.send_message("Cannot find variable: '%s', please specify"%key,True,markup=markup) else: # self.send_message("Cannot find global variable: '%s', please specify"%key) res=self.send_message(descr,True,markup=markup) self.handler.redisconfig.hset(hkey,key,res) return res def send_message(self,msg,feedback=False,markup=None): # print "spawn:%s"%msg self.tg.send_message(self.chatid,msg, reply_to_message_id="",reply_markup=self._processmarkup(markup)) if feedback: self.start_communication() self.event=Event() self.event.wait() return self.returnmsg.text def start_communication(self): self._activate() self.handler.activeCommunications[self.user]=self def stop_communication(self): if self.user in self.handler.activeCommunications: self.handler.activeCommunications.pop(self.user)
class AvailManager(object): SIM_RUNNING = False sim_thread = None def __init__(self): self.updates = [] self.event = Event() def run_sim(self, delay, size): roomids = Room.objects.filter(avail=True).values_list('id',flat=True) #print 'Room ids: %s ' % roomids ids = [] for i in range(0, len(roomids)): ids.append(roomids[i]) random.shuffle(ids) for i in range(0, len(ids)): roomg = Room.objects.filter(id=ids[i]) updateavail(Room.objects.filter(id=ids[i])) room = roomg[0] self.updates.append(RoomUpdate(room)) self.event.set() self.event.clear() if i % size == 0: sleep(delay) self.SIM_RUNNING = False def start_sim(self, delay, size=1): if self.SIM_RUNNING: kill(self.sim_thread) self.updates = [] Room.objects.all().update(avail=True) self.sim_thread = spawn(self.run_sim, delay=delay, size=size) self.SIM_RUNNING = True return HttpResponse('Started Simulation with delay %d' % delay) def stop_sim(self): if not self.SIM_RUNNING: return HttpResponse('No current simulation') kill(self.sim_thread) self.SIM_RUNNING = False return HttpResponse('Stopped simulation') def check_avail(self, timestamp): if len(self.updates) == 0 or timestamp > self.updates[0].timestamp: self.event.wait() room_ids = [] i = len(self.updates) - 1 while i >= 0: i = i - 1 update = self.updates[i] if timestamp <= update.timestamp: room_ids.append(update.room_id) else: break return {'timestamp':int(time.time()), 'rooms':room_ids}
class Console(BaseService): """ A service starting an interactive ipython session when receiving the SIGSTP signal (e.g. via keyboard shortcut CTRL-Z). """ def __init__(self, app): super().__init__(app) self.interrupt = Event() self.console_locals = {} if app.start_console: self.start() self.interrupt.set() else: SigINTHandler(self.interrupt) def start(self): # start console service super().start() class Raiden: def __init__(self, app): self.app = app self.console_locals = dict( _raiden=Raiden(self.app), raiden=self.app.raiden, chain=self.app.raiden.chain, discovery=self.app.discovery, tools=ConsoleTools( self.app.raiden, self.app.discovery, self.app.config['settle_timeout'], self.app.config['reveal_timeout'], ), denoms=denoms, true=True, false=False, usage=print_usage, ) def _run(self): # pylint: disable=method-hidden self.interrupt.wait() print('\n' * 2) print('Entering Console' + OKGREEN) print('Tip:' + OKBLUE) print_usage() # Remove handlers that log to stderr root = getLogger() for handler in root.handlers[:]: if isinstance(handler, StreamHandler) and handler.stream == sys.stderr: root.removeHandler(handler) stream = io.StringIO() handler = StreamHandler(stream=stream) handler.formatter = Formatter(u'%(levelname)s:%(name)s %(message)s') root.addHandler(handler) def lastlog(n=10, prefix=None, level=None): """ Print the last `n` log lines to stdout. Use `prefix='p2p'` to filter for a specific logger. Use `level=INFO` to filter for a specific level. Level- and prefix-filtering are applied before tailing the log. """ lines = (stream.getvalue().strip().split('\n') or []) if prefix: lines = [ line for line in lines if line.split(':')[1].startswith(prefix) ] if level: lines = [line for line in lines if line.split(':')[0] == level] for line in lines[-n:]: print(line) self.console_locals['lastlog'] = lastlog err = io.StringIO() sys.stderr = err def lasterr(n=1): """ Print the last `n` entries of stderr to stdout. """ for line in (err.getvalue().strip().split('\n') or [])[-n:]: print(line) self.console_locals['lasterr'] = lasterr IPython.start_ipython(argv=['--gui', 'gevent'], user_ns=self.console_locals) self.interrupt.clear() sys.exit(0)
class MDSThrasher(Thrasher, Greenlet): """ MDSThrasher:: The MDSThrasher thrashes MDSs during execution of other tasks (workunits, etc). The config is optional. Many of the config parameters are a a maximum value to use when selecting a random value from a range. To always use the maximum value, set no_random to true. The config is a dict containing some or all of: max_thrash: [default: 1] the maximum number of active MDSs per FS that will be thrashed at any given time. max_thrash_delay: [default: 30] maximum number of seconds to delay before thrashing again. max_replay_thrash_delay: [default: 4] maximum number of seconds to delay while in the replay state before thrashing. max_revive_delay: [default: 10] maximum number of seconds to delay before bringing back a thrashed MDS. randomize: [default: true] enables randomization and use the max/min values seed: [no default] seed the random number generator thrash_in_replay: [default: 0.0] likelihood that the MDS will be thrashed during replay. Value should be between 0.0 and 1.0. thrash_max_mds: [default: 0.05] likelihood that the max_mds of the mds cluster will be modified to a value [1, current) or (current, starting max_mds]. Value should be between 0.0 and 1.0. thrash_while_stopping: [default: false] thrash an MDS while there are MDS in up:stopping (because max_mds was changed and some MDS were deactivated). thrash_weights: allows specific MDSs to be thrashed more/less frequently. This option overrides anything specified by max_thrash. This option is a dict containing mds.x: weight pairs. For example, [mds.a: 0.7, mds.b: 0.3, mds.c: 0.0]. Each weight is a value from 0.0 to 1.0. Any MDSs not specified will be automatically given a weight of 0.0 (not thrashed). For a given MDS, by default the trasher delays for up to max_thrash_delay, trashes, waits for the MDS to recover, and iterates. If a non-zero weight is specified for an MDS, for each iteration the thrasher chooses whether to thrash during that iteration based on a random value [0-1] not exceeding the weight of that MDS. Examples:: The following example sets the likelihood that mds.a will be thrashed to 80%, mds.b to 20%, and other MDSs will not be thrashed. It also sets the likelihood that an MDS will be thrashed in replay to 40%. Thrash weights do not have to sum to 1. tasks: - ceph: - mds_thrash: thrash_weights: - mds.a: 0.8 - mds.b: 0.2 thrash_in_replay: 0.4 - ceph-fuse: - workunit: clients: all: [suites/fsx.sh] The following example disables randomization, and uses the max delay values: tasks: - ceph: - mds_thrash: max_thrash_delay: 10 max_revive_delay: 1 max_replay_thrash_delay: 4 """ def __init__(self, ctx, manager, config, fs, max_mds): super(MDSThrasher, self).__init__() self.config = config self.ctx = ctx self.logger = log.getChild('fs.[{f}]'.format(f = fs.name)) self.fs = fs self.manager = manager self.max_mds = max_mds self.name = 'thrasher.fs.[{f}]'.format(f = fs.name) self.stopping = Event() self.randomize = bool(self.config.get('randomize', True)) self.thrash_max_mds = float(self.config.get('thrash_max_mds', 0.05)) self.max_thrash = int(self.config.get('max_thrash', 1)) self.max_thrash_delay = float(self.config.get('thrash_delay', 120.0)) self.thrash_in_replay = float(self.config.get('thrash_in_replay', False)) assert self.thrash_in_replay >= 0.0 and self.thrash_in_replay <= 1.0, 'thrash_in_replay ({v}) must be between [0.0, 1.0]'.format( v=self.thrash_in_replay) self.max_replay_thrash_delay = float(self.config.get('max_replay_thrash_delay', 4.0)) self.max_revive_delay = float(self.config.get('max_revive_delay', 10.0)) def _run(self): try: self.do_thrash() except Exception as e: # Log exceptions here so we get the full backtrace (gevent loses them). # Also allow successful completion as gevent exception handling is a broken mess: # # 2017-02-03T14:34:01.259 CRITICAL:root: File "gevent.libev.corecext.pyx", line 367, in gevent.libev.corecext.loop.handle_error (src/gevent/libev/gevent.corecext.c:5051) # File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/gevent/hub.py", line 558, in handle_error # self.print_exception(context, type, value, tb) # File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/gevent/hub.py", line 605, in print_exception # traceback.print_exception(type, value, tb, file=errstream) # File "/usr/lib/python2.7/traceback.py", line 124, in print_exception # _print(file, 'Traceback (most recent call last):') # File "/usr/lib/python2.7/traceback.py", line 13, in _print # file.write(str+terminator) # 2017-02-03T14:34:01.261 CRITICAL:root:IOError self.set_thrasher_exception(e) self.logger.exception("exception:") # allow successful completion so gevent doesn't see an exception... def log(self, x): """Write data to the logger assigned to MDSThrasher""" self.logger.info(x) def stop(self): self.stopping.set() def kill_mds(self, mds): if self.config.get('powercycle'): (remote,) = (self.ctx.cluster.only('mds.{m}'.format(m=mds)). remotes.keys()) self.log('kill_mds on mds.{m} doing powercycle of {s}'. format(m=mds, s=remote.name)) self._assert_ipmi(remote) remote.console.power_off() else: self.ctx.daemons.get_daemon('mds', mds).stop() @staticmethod def _assert_ipmi(remote): assert remote.console.has_ipmi_credentials, ( "powercycling requested but RemoteConsole is not " "initialized. Check ipmi config.") def revive_mds(self, mds): """ Revive mds -- do an ipmpi powercycle (if indicated by the config) and then restart. """ if self.config.get('powercycle'): (remote,) = (self.ctx.cluster.only('mds.{m}'.format(m=mds)). remotes.keys()) self.log('revive_mds on mds.{m} doing powercycle of {s}'. format(m=mds, s=remote.name)) self._assert_ipmi(remote) remote.console.power_on() self.manager.make_admin_daemon_dir(self.ctx, remote) args = [] self.ctx.daemons.get_daemon('mds', mds).restart(*args) def wait_for_stable(self, rank = None, gid = None): self.log('waiting for mds cluster to stabilize...') for itercount in itertools.count(): status = self.fs.status() max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds'] ranks = list(status.get_ranks(self.fs.id)) stopping = sum(1 for _ in ranks if "up:stopping" == _['state']) actives = sum(1 for _ in ranks if "up:active" == _['state'] and "laggy_since" not in _) if not bool(self.config.get('thrash_while_stopping', False)) and stopping > 0: if itercount % 5 == 0: self.log('cluster is considered unstable while MDS are in up:stopping (!thrash_while_stopping)') else: if rank is not None: try: info = status.get_rank(self.fs.id, rank) if info['gid'] != gid and "up:active" == info['state']: self.log('mds.{name} has gained rank={rank}, replacing gid={gid}'.format(name = info['name'], rank = rank, gid = gid)) return status except: pass # no rank present if actives >= max_mds: # no replacement can occur! self.log("cluster has {actives} actives (max_mds is {max_mds}), no MDS can replace rank {rank}".format( actives=actives, max_mds=max_mds, rank=rank)) return status else: if actives == max_mds: self.log('mds cluster has {count} alive and active, now stable!'.format(count = actives)) return status, None if itercount > 300/2: # 5 minutes raise RuntimeError('timeout waiting for cluster to stabilize') elif itercount % 5 == 0: self.log('mds map: {status}'.format(status=status)) else: self.log('no change') sleep(2) def do_thrash(self): """ Perform the random thrashing action """ self.log('starting mds_do_thrash for fs {fs}'.format(fs = self.fs.name)) stats = { "max_mds": 0, "deactivate": 0, "kill": 0, } while not self.stopping.is_set(): delay = self.max_thrash_delay if self.randomize: delay = random.randrange(0.0, self.max_thrash_delay) if delay > 0.0: self.log('waiting for {delay} secs before thrashing'.format(delay=delay)) self.stopping.wait(delay) if self.stopping.is_set(): continue status = self.fs.status() if random.random() <= self.thrash_max_mds: max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds'] options = [i for i in range(1, self.max_mds + 1) if i != max_mds] if len(options) > 0: new_max_mds = random.choice(options) self.log('thrashing max_mds: %d -> %d' % (max_mds, new_max_mds)) self.fs.set_max_mds(new_max_mds) stats['max_mds'] += 1 self.wait_for_stable() count = 0 for info in status.get_ranks(self.fs.id): name = info['name'] label = 'mds.' + name rank = info['rank'] gid = info['gid'] # if thrash_weights isn't specified and we've reached max_thrash, # we're done count = count + 1 if 'thrash_weights' not in self.config and count > self.max_thrash: break weight = 1.0 if 'thrash_weights' in self.config: weight = self.config['thrash_weights'].get(label, '0.0') skip = random.randrange(0.0, 1.0) if weight <= skip: self.log('skipping thrash iteration with skip ({skip}) > weight ({weight})'.format(skip=skip, weight=weight)) continue self.log('kill {label} (rank={rank})'.format(label=label, rank=rank)) self.kill_mds(name) stats['kill'] += 1 # wait for mon to report killed mds as crashed last_laggy_since = None itercount = 0 while True: status = self.fs.status() info = status.get_mds(name) if not info: break if 'laggy_since' in info: last_laggy_since = info['laggy_since'] break if any([(f == name) for f in status.get_fsmap(self.fs.id)['mdsmap']['failed']]): break self.log( 'waiting till mds map indicates {label} is laggy/crashed, in failed state, or {label} is removed from mdsmap'.format( label=label)) itercount = itercount + 1 if itercount > 10: self.log('mds map: {status}'.format(status=status)) sleep(2) if last_laggy_since: self.log( '{label} reported laggy/crashed since: {since}'.format(label=label, since=last_laggy_since)) else: self.log('{label} down, removed from mdsmap'.format(label=label)) # wait for a standby mds to takeover and become active status = self.wait_for_stable(rank, gid) # wait for a while before restarting old active to become new # standby delay = self.max_revive_delay if self.randomize: delay = random.randrange(0.0, self.max_revive_delay) self.log('waiting for {delay} secs before reviving {label}'.format( delay=delay, label=label)) sleep(delay) self.log('reviving {label}'.format(label=label)) self.revive_mds(name) for itercount in itertools.count(): if itercount > 300/2: # 5 minutes raise RuntimeError('timeout waiting for MDS to revive') status = self.fs.status() info = status.get_mds(name) if info and info['state'] in ('up:standby', 'up:standby-replay', 'up:active'): self.log('{label} reported in {state} state'.format(label=label, state=info['state'])) break self.log( 'waiting till mds map indicates {label} is in active, standby or standby-replay'.format(label=label)) sleep(2) for stat in stats: self.log("stat['{key}'] = {value}".format(key = stat, value = stats[stat]))
class APNSService(BaseService): service_type = 'apns' def __init__(self, sandbox=True, **kwargs): self._send_queue = Queue() self._send_queue_cleared = Event() self._send_greenlet = None self.timeout = INITIAL_TIMEOUT self._feedback_queue = Queue() if "certfile" not in kwargs: raise ValueError(u"Must specify a PEM bundle.") if not os.path.exists(kwargs['certfile']): raise ValueError('PEM bundle file not found') self._sslargs = kwargs self._push_connection = None self._sandbox = sandbox self._error_queue = Queue() self._send_greenlet = None self._error_greenlet = None self._feedback_connection = None self._feedback_greenlet = None self.last_err = None def start(self): """Start the message sending loop.""" if self._send_greenlet is None: self._send_greenlet = gevent.spawn(self.save_err, self._send_loop) def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: logger.info("%s service started" % self.service_type) while True: message = self._send_queue.get() try: self.send_notification(message) except Exception: self.error_sending_notification(message) else: self.timeout = INITIAL_TIMEOUT finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set() except gevent.GreenletExit: pass finally: self._send_greenlet = None logger.info("%s service stopped" % self.service_type) def _check_send_connection(self): if self._push_connection is None: tcp_socket = socket.socket( socket.AF_INET, socket.SOCK_STREAM, 0) s = ssl.wrap_socket(tcp_socket, ssl_version=ssl.PROTOCOL_TLSv1, **self._sslargs) addr = ["gateway.push.apple.com", 2195] if self._sandbox: addr[0] = "gateway.sandbox.push.apple.com" logger.debug('Connecting to %s' % addr[0]) s.connect_ex(tuple(addr)) self._push_connection = s self._error_greenlet = gevent.spawn(self.save_err, self._error_loop) def _check_feedback_connection(self): if self._feedback_connection is None: tcp_socket = socket.socket( socket.AF_INET, socket.SOCK_STREAM, 0) s = ssl.wrap_socket(tcp_socket, ssl_version=ssl.PROTOCOL_TLSv1, **self._sslargs) addr = ["feedback.push.apple.com", 2196] if self._sandbox: addr[0] = "feedback.sandbox.push.apple.com" logger.debug('Connecting to %s' % addr[0]) s.connect_ex(tuple(addr)) self._feedback_connection = s def _error_loop(self): self._error_greenlet = gevent.getcurrent() try: while True: if self._push_connection is None: break msg = self._push_connection.recv(1 + 1 + 4) if len(msg) < 6: return data = struct.unpack("!bbI", msg) self._error_queue.put((data[1], data[2])) except gevent.GreenletExit: logger.exception('Error') finally: if self._push_connection is not None: self._push_connection.close() self._push_connection = None self._error_greenlet = None def _feedback_loop(self): self._feedback_greenlet = gevent.getcurrent() try: self._check_feedback_connection() while True: msg = self._feedback_connection.recv(4 + 2 + 32) if len(msg) < 38: return data = struct.unpack("!IH32s", msg) token = binascii.b2a_hex(data[2]).decode('ascii') self._feedback_queue.put((data[0], token)) except gevent.GreenletExit: logger.exception('Error') finally: if self._feedback_connection: self._feedback_connection.close() self._feedback_connection = None self._feedback_greenlet = None def queue_notification(self, obj): """Send a push notification""" if not isinstance(obj, APNSNotification): raise ValueError(u"You can only send APNSNotification objects.") self._send_queue.put(obj) def get_error(self, block=True, timeout=None): """ Get the next error message. Each error message is a 2-tuple of (status, identifier).""" return self._error_queue.get(block=block, timeout=timeout) def get_feedback(self, block=True, timeout=None): """ Get the next feedback message. Each feedback message is a 2-tuple of (timestamp, device_token).""" if self._feedback_greenlet is None: self._feedback_greenlet = gevent.spawn(self.save_err, self._feedback_loop) return self._feedback_queue.get( block=block, timeout=timeout) def stop(self, timeout=10.0): """ Send all pending messages, close connection. Returns True if no message left to sent. False if dirty. - timeout: seconds to wait for sending remaining messages. disconnect immediately if None. """ if (self._send_greenlet is not None) and \ (self._send_queue.qsize() > 0): self.wait_send(timeout=timeout) if self._send_greenlet is not None: gevent.kill(self._send_greenlet) self._send_greenlet = None if self._error_greenlet is not None: gevent.kill(self._error_greenlet) self._error_greenlet = None if self._feedback_greenlet is not None: gevent.kill(self._feedback_greenlet) self._feedback_greenlet = None return self._send_queue.qsize() < 1 def wait_send(self, timeout=None): self._send_queue_cleared.clear() return self._send_queue_cleared.wait(timeout=timeout) def error_sending_notification(self, notification): if self._push_connection is not None: self._push_connection.close() self._push_connection = None logger.exception("Error while pushing") self._send_queue.put(notification) gevent.sleep(self.timeout) # approaching Fibonacci series timeout = int(round(float(self.timeout) * 1.6)) self.timeout = min(timeout, MAX_TIMEOUT) def send_notification(self, notification): self._check_send_connection() logger.debug('Sending APNS notification') self._push_connection.send(notification.pack()) def save_err(self, func, *args, **kwargs): try: func(*args, **kwargs) except Exception as e: self.last_err = e raise def get_last_error(self): return self.last_err
class Group(object): """Maintain a group of greenlets that are still running. Links to each item and removes it upon notification. """ greenlet_class = Greenlet def __init__(self, *args): assert len(args) <= 1, args self.greenlets = set(*args) if args: for greenlet in args[0]: greenlet.rawlink(self._discard) # each item we kill we place in dying, to avoid killing the same greenlet twice self.dying = set() self._empty_event = Event() self._empty_event.set() def __repr__(self): return '<%s at 0x%x %s>' % (self.__class__.__name__, id(self), self.greenlets) def __len__(self): return len(self.greenlets) def __contains__(self, item): return item in self.greenlets def __iter__(self): return iter(self.greenlets) def add(self, greenlet): try: rawlink = greenlet.rawlink except AttributeError: pass # non-Greenlet greenlet, like MAIN else: rawlink(self._discard) self.greenlets.add(greenlet) self._empty_event.clear() def _discard(self, greenlet): self.greenlets.discard(greenlet) self.dying.discard(greenlet) if not self.greenlets: self._empty_event.set() def discard(self, greenlet): self._discard(greenlet) try: unlink = greenlet.unlink except AttributeError: pass # non-Greenlet greenlet, like MAIN else: unlink(self._discard) def start(self, greenlet): self.add(greenlet) greenlet.start() def spawn(self, *args, **kwargs): greenlet = self.greenlet_class(*args, **kwargs) self.start(greenlet) return greenlet # def close(self): # """Prevents any more tasks from being submitted to the pool""" # self.add = RaiseException("This %s has been closed" % self.__class__.__name__) def join(self, timeout=None, raise_error=False): if raise_error: greenlets = self.greenlets.copy() self._empty_event.wait(timeout=timeout) for greenlet in greenlets: if greenlet.exception is not None: raise greenlet.exception else: self._empty_event.wait(timeout=timeout) def kill(self, exception=GreenletExit, block=True, timeout=None): timer = Timeout.start_new(timeout) try: try: while self.greenlets: for greenlet in list(self.greenlets): if greenlet not in self.dying: try: kill = greenlet.kill except AttributeError: _kill(greenlet, exception) else: kill(exception, block=False) self.dying.add(greenlet) if not block: break joinall(self.greenlets) except Timeout: ex = sys.exc_info()[1] if ex is not timer: raise finally: timer.cancel() def killone(self, greenlet, exception=GreenletExit, block=True, timeout=None): if greenlet not in self.dying and greenlet in self.greenlets: greenlet.kill(exception, block=False) self.dying.add(greenlet) if block: greenlet.join(timeout) def apply(self, func, args=None, kwds=None): """Equivalent of the apply() builtin function. It blocks till the result is ready.""" if args is None: args = () if kwds is None: kwds = {} if getcurrent() in self: return func(*args, **kwds) else: return self.spawn(func, *args, **kwds).get() def apply_cb(self, func, args=None, kwds=None, callback=None): result = self.apply(func, args, kwds) if callback is not None: Greenlet.spawn(callback, result) return result def apply_async(self, func, args=None, kwds=None, callback=None): """A variant of the apply() method which returns a Greenlet object. If callback is specified then it should be a callable which accepts a single argument. When the result becomes ready callback is applied to it (unless the call failed).""" if args is None: args = () if kwds is None: kwds = {} if self.full(): # cannot call spawn() directly because it will block return Greenlet.spawn(self.apply_cb, func, args, kwds, callback) else: greenlet = self.spawn(func, *args, **kwds) if callback is not None: greenlet.link(pass_value(callback)) return greenlet def map(self, func, iterable): return list(self.imap(func, iterable)) def map_cb(self, func, iterable, callback=None): result = self.map(func, iterable) if callback is not None: callback(result) return result def map_async(self, func, iterable, callback=None): """ A variant of the map() method which returns a Greenlet object. If callback is specified then it should be a callable which accepts a single argument. """ return Greenlet.spawn(self.map_cb, func, iterable, callback) def imap(self, func, iterable): """An equivalent of itertools.imap()""" return IMap.spawn(func, iterable, spawn=self.spawn) def imap_unordered(self, func, iterable): """The same as imap() except that the ordering of the results from the returned iterator should be considered in arbitrary order.""" return IMapUnordered.spawn(func, iterable, spawn=self.spawn) def full(self): return False def wait_available(self): pass
def test_replay_with_parameters(self): #-------------------------------------------------------------------------------- # Create the configurations and the dataset #-------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append( self.dataset_management.create_parameter_context( 'binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append( self.dataset_management.create_parameter_context( 'records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition( 'replay_stream', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream( 'replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id) es.start() self.addCleanup(es.stop) self.publish_fake_data(stream_id, route) self.assertTrue(dataset_modified.wait(30)) query = { 'start_time': 0 - 2208988800, 'end_time': 20 - 2208988800, 'stride_time': 2, 'parameters': ['time', 'temp'] } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id, query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(0, 20, 2) == rdt['time'] self.assertTrue(comp.all(), '%s' % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(['time', 'temp'])) extents = self.dataset_management.dataset_extents( dataset_id=dataset_id, parameters=['time', 'temp']) self.assertTrue(extents['time'] >= 20) self.assertTrue(extents['temp'] >= 20) self.streams.append(stream_id) self.stop_ingestion(stream_id)
class Queue(Greenlet): """Manages the queue of |Envelope| objects waiting for delivery. This is not a standard FIFO queue, a message's place in the queue depends entirely on the timestamp of its next delivery attempt. :param store: Object implementing :class:`QueueStorage`. :param relay: |Relay| object used to attempt message deliveries. If this is not given, no deliveries will be attempted on received messages. :param backoff: Function that, given an |Envelope| and number of delivery attempts, will return the number of seconds before the next attempt. If it returns ``None``, the message will be permanently failed. The default backoff function simply returns ``None`` and messages are never retried. :param bounce_factory: Function that produces a |Bounce| object given the same parameters as the |Bounce| constructor. If the function returns ``None``, no bounce is delivered. By default, a new |Bounce| is created in every case. :param bounce_queue: |Queue| object that will be used for delivering bounce messages. The default is ``self``. :param store_pool: Number of simultaneous operations performable against the ``store`` object. Default is unlimited. :param relay_pool: Number of simultaneous operations performable against the ``relay`` object. Default is unlimited. """ def __init__(self, store, relay=None, backoff=None, bounce_factory=None, bounce_queue=None, store_pool=None, relay_pool=None): super(Queue, self).__init__() self.store = store self.relay = relay self.backoff = backoff or self._default_backoff self.bounce_factory = bounce_factory or Bounce self.bounce_queue = bounce_queue or self self.wake = Event() self.queued = [] self.active_ids = set() self.queued_ids = set() self.queued_lock = Semaphore(1) self.queue_policies = [] self._use_pool('store_pool', store_pool) self._use_pool('relay_pool', relay_pool) def add_policy(self, policy): """Adds a |QueuePolicy| to be executed before messages are persisted to storage. :param policy: |QueuePolicy| object to execute. """ if isinstance(policy, QueuePolicy): self.queue_policies.append(policy) else: raise TypeError('Argument not a QueuePolicy.') @staticmethod def _default_backoff(envelope, attempts): pass def _run_policies(self, envelope): results = [envelope] def recurse(current, i): try: policy = self.queue_policies[i] except IndexError: return ret = policy.apply(current) if ret: results.remove(current) results.extend(ret) for env in ret: recurse(env, i + 1) else: recurse(current, i + 1) recurse(envelope, 0) return results def _use_pool(self, attr, pool): if pool is None: pass elif isinstance(pool, Pool): setattr(self, attr, pool) else: setattr(self, attr, Pool(pool)) def _pool_run(self, which, func, *args, **kwargs): pool = getattr(self, which + '_pool', None) if pool: ret = pool.spawn(func, *args, **kwargs) return ret.get() else: return func(*args, **kwargs) def _pool_imap(self, which, func, *iterables): pool = getattr(self, which + '_pool', gevent) threads = imap(pool.spawn, repeat(func), *iterables) ret = [] for thread in threads: thread.join() ret.append(thread.exception or thread.value) return ret def _pool_spawn(self, which, func, *args, **kwargs): pool = getattr(self, which + '_pool', gevent) return pool.spawn(func, *args, **kwargs) def _add_queued(self, entry): timestamp, id = entry if id not in self.queued_ids | self.active_ids: bisect.insort(self.queued, entry) self.queued_ids.add(id) self.wake.set() def enqueue(self, envelope): """Drops a new message in the queue for delivery. The first delivery attempt is made immediately (depending on relay pool availability). This method is not typically called directly, |Edge| objects use it when they receive new messages. :param envelope: |Envelope| object to enqueue. :returns: Zipped list of envelopes and their respective queue IDs (or thrown :exc:`QueueError` objects). """ now = time.time() envelopes = self._run_policies(envelope) ids = self._pool_imap('store', self.store.write, envelopes, repeat(now)) results = zip(envelopes, ids) for env, id in results: if not isinstance(id, BaseException): if self.relay: self.active_ids.add(id) self._pool_spawn('relay', self._attempt, id, env, 0) elif not isinstance(id, QueueError): raise id # Re-raise exceptions that are not QueueError. return results def _load_all(self): for entry in self.store.load(): self._add_queued(entry) def _remove(self, id): self._pool_spawn('store', self.store.remove, id) self.queued_ids.discard(id) self.active_ids.discard(id) def _bounce(self, envelope, reply): bounce = self.bounce_factory(envelope, reply) if bounce: return self.bounce_queue.enqueue(bounce) def _perm_fail(self, id, envelope, reply): if id is not None: self._remove(id) if envelope.sender: # Can't bounce to null-sender. self._pool_spawn('bounce', self._bounce, envelope, reply) def _split_by_reply(self, envelope, replies): if isinstance(replies, Reply): return [(replies, envelope)] groups = [] for i, rcpt in enumerate(envelope.recipients): for reply, group_env in groups: if replies[i] == reply: group_env.recipients.append(rcpt) break else: group_env = envelope.copy([rcpt]) groups.append((replies[i], group_env)) return groups def _retry_later(self, id, envelope, replies): attempts = self.store.increment_attempts(id) wait = self.backoff(envelope, attempts) if wait is None: for reply, group_env in self._split_by_reply(envelope, replies): reply.message += ' (Too many retries)' self._perm_fail(None, group_env, reply) self._remove(id) return False else: when = time.time() + wait self.store.set_timestamp(id, when) self.active_ids.discard(id) self._add_queued((when, id)) return True def _attempt(self, id, envelope, attempts): try: results = self.relay._attempt(envelope, attempts) except TransientRelayError as e: self._pool_spawn('store', self._retry_later, id, envelope, e.reply) except PermanentRelayError as e: self._perm_fail(id, envelope, e.reply) except Exception as e: log_exception(__name__) reply = Reply('450', '4.0.0 Unhandled delivery error: ' + str(e)) self._pool_spawn('store', self._retry_later, id, envelope, reply) raise else: if isinstance(results, collections.Sequence): self._handle_partial_relay(id, envelope, attempts, results) else: self._remove(id) def _handle_partial_relay(self, id, envelope, attempts, results): delivered = [] tempfails = [] permfails = [] for i, rcpt in enumerate(envelope.recipients): rcpt_res = results[i] if not rcpt_res: delivered.append(i) elif isinstance(rcpt_res, PermanentRelayError): delivered.append(i) permfails.append((rcpt, rcpt_res.reply)) elif isinstance(rcpt_res, TransientRelayError): tempfails.append((rcpt, rcpt_res.reply)) if permfails: rcpts, replies = zip(*permfails) fail_env = envelope.copy(rcpts) for reply, group_env in self._split_by_reply(fail_env, replies): self._perm_fail(None, group_env, reply) if tempfails: rcpts, replies = zip(*tempfails) fail_env = envelope.copy(rcpts) if not self._retry_later(id, fail_env, replies): return else: self.store.remove(id) return self.store.set_recipients_delivered(id, delivered) def _dequeue(self, id): try: envelope, attempts = self.store.get(id) except KeyError: return self.active_ids.add(id) self._pool_spawn('relay', self._attempt, id, envelope, attempts) def _check_ready(self, now): last_i = 0 for i, entry in enumerate(self.queued): timestamp, id = entry if now >= timestamp: self._pool_spawn('store', self._dequeue, id) last_i = i + 1 else: break if last_i > 0: self.queued = self.queued[last_i:] self.queued_ids = set([id for timestamp, id in self.queued]) def _wait_store(self): while True: try: for entry in self.store.wait(): self._add_queued(entry) except NotImplementedError: return def _wait_ready(self, now): try: first = self.queued[0] except IndexError: self.wake.wait() self.wake.clear() return first_timestamp = first[0] if first_timestamp > now: self.wake.wait(first_timestamp - now) self.wake.clear() def flush(self): """Attempts to immediately flush all messages waiting in the queue, regardless of their retry timers. .. warning:: This can be a very expensive operation, use with care. """ self.wake.set() self.wake.clear() self.queued_lock.acquire() try: for entry in self.queued: self._pool_spawn('store', self._dequeue, entry[1]) self.queued = [] finally: self.queued_lock.release() def kill(self): """This method is used by |Queue| and |Queue|-like objects to properly end any associated services (such as running :class:`~gevent.Greenlet` threads) and close resources. """ super(Queue, self).kill() def _run(self): if not self.relay: return self._pool_spawn('store', self._load_all) self._pool_spawn('store', self._wait_store) while True: self.queued_lock.acquire() try: now = time.time() self._check_ready(now) self._wait_ready(now) finally: self.queued_lock.release()
class Session: def __init__(self, handler, tg, chatid, user, name): self.handler = handler self.tg = tg self.name = name self.user = user self.chatid = chatid self.event = None self.returnmsg = None def _activate(self): self.handler.activeSessions[self.user] = self def _processmarkup(self, markup): if markup != None: markup2 = {} markup2["resize_keyboard"] = True markup2["one_time_keyboard"] = True markup2["keyboard"] = markup return json.dumps(markup2) return markup def getArgument(self, key, descr="", global_=False, markup=None, presetvalue=None): if global_: hkey = "config" else: hkey = self.name res = self.handler.redisconfig.hget(hkey, key) if res == None: # if descr=="": # session.send_message("Cannot find global variable: '%s' please define by using '%s=yourinput'."%(key,key)) # else: if presetvalue != None: res = presetvalue elif descr == "": res = self.send_message( "Cannot find variable: '%s', please specify" % key, True, markup=markup) else: # self.send_message("Cannot find global variable: '%s', please specify"%key) res = self.send_message(descr, True, markup=markup) self.handler.redisconfig.hset(hkey, key, res) return res def send_message(self, msg, feedback=False, markup=None): # print "spawn:%s"%msg self.tg.send_message(self.chatid, msg, reply_to_message_id="", reply_markup=self._processmarkup(markup)) if feedback: self.start_communication() self.event = Event() self.event.wait() return self.returnmsg.text def start_communication(self): self._activate() self.handler.activeCommunications[self.user] = self def stop_communication(self): if self.handler.activeCommunications.has_key(self.user): self.handler.activeCommunications.pop(self.user)
class InboundESL(object): def __init__(self, host, port, password): self.host = host self.port = port self.password = password self.timeout = 5 self._run = True self._EOL = '\n' self._commands_sent = [] self._auth_request_event = Event() self._receive_events_greenlet = None self._process_events_greenlet = None self.event_handlers = {} self.connected = False self._esl_event_queue = Queue() self._process_esl_event_queue = True def connect(self): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.settimeout(self.timeout) self.sock.connect((self.host, self.port)) self.connected = True self.sock.settimeout(None) self.sock_file = self.sock.makefile() self._receive_events_greenlet = gevent.spawn(self.receive_events) self._process_events_greenlet = gevent.spawn(self.process_events) self._auth_request_event.wait() self.authenticate() def receive_events(self): buf = '' while self._run: try: data = self.sock_file.readline() except Exception: self._run = False self.connected = False self.sock.close() # logging.exception("Error reading from socket.") break if not data: if self.connected: logging.error( "Error receiving data, is FreeSWITCH running?") self.connected = False break # Empty line if data == self._EOL: event = ESLEvent(buf) buf = '' self.handle_event(event) continue buf += data @staticmethod def _read_socket(sock, length): """Receive data from socket until the length is reached.""" data = sock.read(length) data_length = len(data) while data_length < length: logging.warn( 'Socket should read %s bytes, but actually read %s bytes. ' 'Consider increasing "net.core.rmem_default".' % (length, data_length)) # FIXME(italo): if not data raise error data += sock.read(length - data_length) data_length = len(data) return data def handle_event(self, event): if event.headers['Content-Type'] == 'auth/request': self._auth_request_event.set() elif event.headers['Content-Type'] == 'command/reply': async_response = self._commands_sent.pop(0) event.data = event.headers['Reply-Text'] async_response.set(event) elif event.headers['Content-Type'] == 'api/response': length = int(event.headers['Content-Length']) data = self._read_socket(self.sock_file, length) event.data = data async_response = self._commands_sent.pop(0) async_response.set(event) elif event.headers['Content-Type'] == 'text/disconnect-notice': self.connected = False else: length = int(event.headers['Content-Length']) data = self._read_socket(self.sock_file, length) if event.headers.get('Content-Type') == 'log/data': event.data = data else: event.parse_data(data) self._esl_event_queue.put(event) def _safe_exec_handler(self, handler, event): try: handler(event) except: logging.exception('ESL %s raised exception.' % handler.__name__) logging.error(pprint.pformat(event.headers)) def process_events(self): logging.debug('Event Processor Running') while self._run: if not self._process_esl_event_queue: gevent.sleep(1) continue try: event = self._esl_event_queue.get(timeout=1) except gevent.queue.Empty: continue if event.headers.get('Event-Name') == 'CUSTOM': handlers = self.event_handlers.get( event.headers.get('Event-Subclass')) else: handlers = self.event_handlers.get( event.headers.get('Event-Name')) if not handlers and event.headers.get( 'Content-Type') == 'log/data': handlers = self.event_handlers.get('log') if not handlers: continue if hasattr(self, 'before_handle'): self._safe_exec_handler(self.before_handle, event) for handle in handlers: self._safe_exec_handler(handle, event) if hasattr(self, 'after_handle'): self._safe_exec_handler(self.after_handle, event) def send(self, data): if not self.connected: raise NotConnectedError() async_response = gevent.event.AsyncResult() self._commands_sent.append(async_response) raw_msg = (data + self._EOL * 2).encode('utf-8') self.sock.send(raw_msg) response = async_response.get() return response def authenticate(self): response = self.send('auth %s' % self.password) if response.headers['Reply-Text'] != '+OK accepted': raise ValueError('Invalid password.') def register_handle(self, name, handler): if name not in self.event_handlers: self.event_handlers[name] = [] if handler in self.event_handlers[name]: return self.event_handlers[name].append(handler) def unregister_handle(self, name, handler): if name not in self.event_handlers: raise ValueError('No handlers found for event: %s' % name) self.event_handlers[name].remove(handler) if not self.event_handlers[name]: del self.event_handlers[name] def stop(self): if self.connected: self.send('exit') self._run = False logging.info("Waiting for receive greenlet exit") self._receive_events_greenlet.join() logging.info("Waiting for event processing greenlet exit") self._process_events_greenlet.join() if self.connected: self.sock.close() self.sock_file.close()
class RotkehlchenServer(): def __init__(self): arg_parser = app_args( prog='rotkehlchen', description='Rotkehlchen Crypto Portfolio Management', ) self.args = arg_parser.parse_args() self.rotkehlchen = Rotkehlchen(self.args) self.stop_event = Event() mainloop_greenlet = self.rotkehlchen.start() mainloop_greenlet.link_exception(self.handle_killed_greenlets) # Greenlets that will be waited for when we shutdown self.waited_greenlets = [mainloop_greenlet] # Greenlets that can be killed instead of waited for when we shutdown self.killable_greenlets = [] self.task_lock = Semaphore() self.task_id = 0 self.task_results = {} def new_task_id(self): with self.task_lock: task_id = self.task_id self.task_id += 1 return task_id def write_task_result(self, task_id, result): with self.task_lock: self.task_results[task_id] = result def get_task_result(self, task_id): with self.task_lock: return self.task_results[task_id] def port(self): return self.args.zerorpc_port def shutdown(self): log.debug('Shutdown initiated') self.zerorpc.stop() self.rotkehlchen.shutdown() log.debug('Waiting for greenlets') gevent.wait(self.waited_greenlets) log.debug('Waited for greenlets. Killing all other greenlets') gevent.killall(self.killable_greenlets) log.debug('Greenlets killed. Killing zerorpc greenlet') self.zerorpc_greenlet.kill() log.debug('Killed zerorpc greenlet') log.debug('Shutdown completed') logging.shutdown() self.stop_event.set() def logout(self): # Kill all queries apart from the main loop -- perhaps a bit heavy handed # but the other options would be: # 1. to wait for all of them. That could take a lot of time, for no reason. # All results would be discarded anyway since we are logging out. # 2. Have an intricate stop() notification system for each greenlet, but # that is going to get complicated fast. gevent.killall(self.killable_greenlets) with self.task_lock: self.task_results = {} self.rotkehlchen.logout() def set_main_currency(self, currency_text): self.rotkehlchen.set_main_currency(currency_text) def set_settings(self, settings): result, message = self.rotkehlchen.set_settings(settings) return {'result': result, 'message': message} def handle_killed_greenlets(self, greenlet): if not greenlet.exception: log.warning('handle_killed_greenlets without an exception') return log.error( 'Greenlet for task {} dies with exception: {}.\n' 'Exception Name: {}\nException Info: {}\nTraceback:\n {}'.format( greenlet.task_id, greenlet.exception, greenlet.exc_info[0], greenlet.exc_info[1], ''.join(traceback.format_tb(greenlet.exc_info[2])), )) # also write an error for the task result result = { 'error': str(greenlet.exception), } self.write_task_result(greenlet.task_id, result) def _query_async(self, command, task_id, **kwargs): result = getattr(self, command)(**kwargs) self.write_task_result(task_id, result) def query_async(self, command, **kwargs): task_id = self.new_task_id() log.debug("NEW TASK {} (kwargs:{}) with ID: {}".format( command, kwargs, task_id)) greenlet = gevent.spawn( self._query_async, command, task_id, **kwargs, ) greenlet.task_id = task_id greenlet.link_exception(self.handle_killed_greenlets) self.killable_greenlets.append(greenlet) return task_id def query_task_result(self, task_id): with self.task_lock: len1 = len(self.task_results) ret = self.task_results.pop(int(task_id), None) if not ret and len1 != len(self.task_results): log.error("Popped None from results task but lost an entry") if ret: log.debug("Found response for task {}".format(task_id)) return ret @staticmethod def get_fiat_exchange_rates(currencies): rates = Inquirer().get_fiat_usd_exchange_rates(currencies) res = {'exchange_rates': rates} return process_result(res) def get_settings(self): return process_result(self.rotkehlchen.data.db.get_settings()) def remove_exchange(self, name): result, message = self.rotkehlchen.remove_exchange(name) return {'result': result, 'message': message} def setup_exchange(self, name, api_key, api_secret): result, message = self.rotkehlchen.setup_exchange( name, api_key, api_secret) return {'result': result, 'message': message} def query_otctrades(self): trades = self.rotkehlchen.data.get_external_trades() result = {'result': trades, 'message': ''} return process_result(result) def add_otctrade(self, data): result, message = self.rotkehlchen.data.add_external_trade(data) return {'result': result, 'message': message} def edit_otctrade(self, data): result, message = self.rotkehlchen.data.edit_external_trade(data) return {'result': result, 'message': message} def delete_otctrade(self, trade_id): result, message = self.rotkehlchen.data.delete_external_trade(trade_id) return {'result': result, 'message': message} def set_premium_credentials(self, api_key, api_secret): msg = '' result = False try: self.rotkehlchen.set_premium_credentials(api_key, api_secret) result = True except (AuthenticationError, IncorrectApiKeyFormat) as e: msg = str(e) return {'result': result, 'message': msg} def set_premium_option_sync(self, should_sync): self.rotkehlchen.data.db.update_premium_sync(should_sync) return True def query_exchange_balances(self, name): res = {'name': name} balances, msg = getattr(self.rotkehlchen, name).query_balances() if balances is None: res['error'] = msg else: res['balances'] = balances return process_result(res) def query_exchange_balances_async(self, name): res = self.query_async('query_exchange_balances', name=name) return {'task_id': res} def query_blockchain_balances(self): result, empty_or_error = self.rotkehlchen.blockchain.query_balances() return process_result({'result': result, 'message': empty_or_error}) def query_blockchain_balances_async(self): res = self.query_async('query_blockchain_balances') return {'task_id': res} def query_fiat_balances(self): res = self.rotkehlchen.query_fiat_balances() return process_result(res) def query_netvalue_data(self): res = self.rotkehlchen.data.db.get_netvalue_data() result = {'times': res[0], 'data': res[1]} return process_result(result) def query_timed_balances_data(self, asset: str, start_ts: int, end_ts: int): start_ts = Timestamp(start_ts) end_ts = Timestamp(end_ts) res = self.rotkehlchen.data.db.query_timed_balances( from_ts=start_ts, to_ts=end_ts, asset=asset, ) result = {'result': res, 'messsage': ''} return process_result(result) def query_owned_assets(self): res = self.rotkehlchen.data.db.query_owned_assets() result = {'result': res, 'message': ''} return process_result(result) def query_latest_location_value_distribution(self): res = self.rotkehlchen.data.db.get_latest_location_value_distribution() result = {'result': res, 'message': ''} return process_result(result) def query_latest_asset_value_distribution(self): res = self.rotkehlchen.data.db.get_latest_asset_value_distribution() result = {'result': res, 'message': ''} return process_result(result) def consume_messages(self): """Consumes all errors and warnings from the messages aggregator""" warnings = self.rotkehlchen.msg_aggregator.consume_warnings() errors = self.rotkehlchen.msg_aggregator.consume_errors() result = { 'result': { 'warnings': warnings, 'errors': errors }, 'message': '' } return process_result(result) def query_statistics_renderer(self): result_dict = {'result': '', 'message': 'user does not have premium'} if not self.rotkehlchen.premium: return process_result(result_dict) active = self.rotkehlchen.premium.is_active() if not active: return process_result(result_dict) try: result = self.rotkehlchen.premium.query_statistics_renderer() result_dict['result'] = result result_dict['message'] = '' except RemoteError as e: result_dict['message'] = str(e) return process_result(result_dict) def set_fiat_balance(self, currency: str, balance: str): result, message = self.rotkehlchen.data.set_fiat_balance( currency, balance) return {'result': result, 'message': message} def query_trade_history(self, location: str, start_ts: int, end_ts: int): start_ts = Timestamp(start_ts) end_ts = Timestamp(end_ts) if location == 'all': return self.rotkehlchen.trades_historian.get_history( start_ts, end_ts) try: exchange = getattr(self.rotkehlchen, location) except AttributeError: raise ValueError(f'Unknown location {location} given') return process_result_list( exchange.query_trade_history(start_ts, end_ts, end_ts)) def process_trade_history(self, start_ts, end_ts): start_ts = int(start_ts) end_ts = int(end_ts) result, error_or_empty = self.rotkehlchen.process_history( start_ts, end_ts) response = {'result': result, 'message': error_or_empty} return process_result(response) def process_trade_history_async(self, start_ts, end_ts): res = self.query_async('process_trade_history', start_ts=start_ts, end_ts=end_ts) return {'task_id': res} def export_processed_history_csv(self, dirpath): result, message = self.rotkehlchen.accountant.csvexporter.create_files( dirpath) return {'result': result, 'message': message} def query_balances(self, save_data=False): if isinstance(save_data, str) and (save_data == 'save' or save_data == 'True'): save_data = True result = self.rotkehlchen.query_balances(save_data) print(pretty_json_dumps(result)) return process_result(result) def query_balances_async(self, save_data=False): res = self.query_async('query_balances', save_data=save_data) return {'task_id': res} def query_periodic_data(self): """Will query for some client data that can change frequently""" result = self.rotkehlchen.query_periodic_data() return process_result(result) def get_eth_tokens(self): result = { 'all_eth_tokens': self.rotkehlchen.data.eth_tokens, 'owned_eth_tokens': self.rotkehlchen.blockchain.eth_tokens, } return process_result(result) def add_owned_eth_tokens(self, tokens): return self.rotkehlchen.add_owned_eth_tokens(tokens) def remove_owned_eth_tokens(self, tokens): return self.rotkehlchen.remove_owned_eth_tokens(tokens) def add_blockchain_account(self, given_blockchain: str, given_account: str): try: blockchain = SupportedBlockchain(given_blockchain) except ValueError: msg = f'Tried to add blockchain account for unsupported blockchain {given_blockchain}' return simple_result(False, msg) return self.rotkehlchen.add_blockchain_account(blockchain, given_account) def remove_blockchain_account(self, given_blockchain: str, given_account: str): try: blockchain = SupportedBlockchain(given_blockchain) except ValueError: msg = ( f'Tried to remove blockchain account for unsupported blockchain {given_blockchain}' ) return simple_result(False, msg) return self.rotkehlchen.remove_blockchain_account( blockchain, given_account) def get_ignored_assets(self): result = { 'ignored_assets': [ identifier for identifier in self.rotkehlchen.data.db.get_ignored_assets() ], } return result def add_ignored_asset(self, asset: str): result, message = self.rotkehlchen.data.add_ignored_asset(asset) return {'result': result, 'message': message} def remove_ignored_asset(self, asset: str): result, message = self.rotkehlchen.data.remove_ignored_asset(asset) return {'result': result, 'message': message} def unlock_user(self, user, password, create_new, sync_approval, api_key, api_secret): """Either unlock an existing user or create a new one""" res = {'result': True, 'message': ''} assert isinstance(sync_approval, str), "sync_approval should be a string" assert isinstance(api_key, str), "api_key should be a string" assert isinstance(api_secret, str), "api_secret should be a string" if not isinstance(create_new, bool): if not isinstance(create_new, str): raise ValueError('create_new can only be boolean or str') if create_new in ('False', 'false', 'FALSE'): create_new = False elif create_new in ('True', 'true', 'TRUE'): create_new = True else: raise ValueError( f'Invalid string value for create_new {create_new}') valid_actions = ['unknown', 'yes', 'no'] valid_approve = isinstance(sync_approval, str) and sync_approval in valid_actions if not valid_approve: raise ValueError('Provided invalid value for sync_approval') if api_key != '' and create_new is False: raise ValueError( 'Should not ever have api_key provided during a normal login') if api_key != '' and api_secret == '' or api_secret != '' and api_key == '': raise ValueError('Must provide both or neither of api key/secret') try: self.rotkehlchen.unlock_user( user, password, create_new, sync_approval, api_key, api_secret, ) res['exchanges'] = self.rotkehlchen.connected_exchanges res['premium'] = self.rotkehlchen.premium is not None res['settings'] = self.rotkehlchen.data.db.get_settings() except AuthenticationError as e: res['result'] = False res['message'] = str(e) except RotkehlchenPermissionError as e: res['result'] = False res['permission_needed'] = True res['message'] = str(e) return res def main(self): if os.name != 'nt': gevent.hub.signal(signal.SIGQUIT, self.shutdown) gevent.hub.signal(signal.SIGINT, self.shutdown) gevent.hub.signal(signal.SIGTERM, self.shutdown) # self.zerorpc = zerorpc.Server(self, heartbeat=15) self.zerorpc = zerorpc.Server(self) addr = 'tcp://127.0.0.1:' + str(self.port()) self.zerorpc.bind(addr) print('start running on {}'.format(addr)) self.zerorpc_greenlet = gevent.spawn(self.zerorpc.run) self.stop_event.wait()
class InstanceRunner(object): """Maintainer for a single instance.""" def __init__(self, app, log_queue, name=None, ppid=0, timeout=30, uid=None, gid=None): self.app = app self.log_queue = log_queue self.name = name or self.__class__.__name__ self.ppid = ppid self.uid = uid or os.geteuid() self.gid = gid or os.getegid() self.timeout = timeout self.booted = False self.tmp = NotifyFile(uid, gid) self.shutdown = Event() self.log = logging.getLogger('child') def run(self): """Main-loop of the instance runner.""" self.init_process() self.log.info("Booting child with pid: %d", os.getpid()) self.app.start() self.booted = True while not self.shutdown.is_set(): self.update_proc_title() if os.getppid() != self.ppid: # Parent changed - lets drop out break self.tmp.notify() self.shutdown.wait(1) self.app.stop() def update_proc_title(self): util._setproctitle('worker %s: %s' % ( self.name, str(self.app), )) def capture_stdout(self): """Setup so that stdout and stderr are sent to a logger.""" sys.stdout = StreamToLogger(logging.getLogger('sys.stdout'), logging.INFO) sys.stderr = StreamToLogger(logging.getLogger('sys.stderr'), logging.ERROR) def init_process(self): """Initialize process.""" random.seed() # FIXME: seed with pid? # Initialize logging. logger = logging.getLogger() logger.addHandler(ChildLogHandler(self.log_queue)) for handler in logger.handlers: if not isinstance(handler, ChildLogHandler): logger.removeHandler(handler) self.capture_stdout() # Initialize the rest. util.close_on_exec(self.tmp.fileno()) self.init_signals() def init_signals(self): signal.signal(signal.SIGINT, signal.SIG_IGN) gevent.signal(signal.SIGQUIT, self.handle_quit) gevent.signal(signal.SIGTERM, self.handle_exit) gevent.signal(signal.SIGWINCH, self.handle_winch) def handle_quit(self): self.log.info('Received quit') self.shutdown.set() def handle_exit(self): self.log.info('Received exit') sys.exit(0) def handle_winch(self): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. return
def healthcheck( transport: UDPTransport, recipient: typing.Address, event_stop: Event, event_healthy: Event, event_unhealthy: Event, nat_keepalive_retries: int, nat_keepalive_timeout: int, nat_invitation_timeout: int, ping_nonce: int, ): """ Sends a periodical Ping to `recipient` to check its health. """ # pylint: disable=too-many-branches log.debug( 'starting healthcheck for', node=pex(transport.raiden.address), to=pex(recipient), ) # The state of the node is unknown, the events are set to allow the tasks # to do work. last_state = NODE_NETWORK_UNKNOWN transport.set_node_network_state( recipient, last_state, ) # Always call `clear` before `set`, since only `set` does context-switches # it's easier to reason about tasks that are waiting on both events. # Wait for the end-point registration or for the node to quit try: transport.get_host_port(recipient) except UnknownAddress: log.debug( 'waiting for endpoint registration', node=pex(transport.raiden.address), to=pex(recipient), ) event_healthy.clear() event_unhealthy.set() backoff = udp_utils.timeout_exponential_backoff( nat_keepalive_retries, nat_keepalive_timeout, nat_invitation_timeout, ) sleep = next(backoff) while not event_stop.wait(sleep): try: transport.get_host_port(recipient) except UnknownAddress: sleep = next(backoff) else: break # Don't wait to send the first Ping and to start sending messages if the # endpoint is known sleep = 0 event_unhealthy.clear() event_healthy.set() while not event_stop.wait(sleep): sleep = nat_keepalive_timeout ping_nonce['nonce'] += 1 messagedata = transport.get_ping(ping_nonce['nonce']) message_id = ('ping', ping_nonce['nonce'], recipient) # Send Ping a few times before setting the node as unreachable try: acknowledged = udp_utils.retry( transport, messagedata, message_id, recipient, event_stop, [nat_keepalive_timeout] * nat_keepalive_retries, ) except RaidenShuttingDown: # For a clean shutdown process return if event_stop.is_set(): return if not acknowledged: log.debug( 'node is unresponsive', node=pex(transport.raiden.address), to=pex(recipient), current_state=last_state, new_state=NODE_NETWORK_UNREACHABLE, retries=nat_keepalive_retries, timeout=nat_keepalive_timeout, ) # The node is not healthy, clear the event to stop all queue # tasks last_state = NODE_NETWORK_UNREACHABLE transport.set_node_network_state( recipient, last_state, ) event_healthy.clear() event_unhealthy.set() # Retry until recovery, used for: # - Checking node status. # - Nat punching. try: acknowledged = udp_utils.retry( transport, messagedata, message_id, recipient, event_stop, repeat(nat_invitation_timeout), ) except RaidenShuttingDown: # For a clean shutdown process return if acknowledged: current_state = views.get_node_network_status( views.state_from_raiden(transport.raiden), recipient, ) log.debug( 'node answered', node=pex(transport.raiden.address), to=pex(recipient), current_state=current_state, new_state=NODE_NETWORK_REACHABLE, ) if last_state != NODE_NETWORK_REACHABLE: last_state = NODE_NETWORK_REACHABLE transport.set_node_network_state( recipient, last_state, ) event_unhealthy.clear() event_healthy.set()
class JoinableQueue(Queue): """ A subclass of :class:`Queue` that additionally has :meth:`task_done` and :meth:`join` methods. """ def __init__(self, maxsize=None, items=None, unfinished_tasks=None): """ .. versionchanged:: 1.1a1 If *unfinished_tasks* is not given, then all the given *items* (if any) will be considered unfinished. """ from gevent.event import Event Queue.__init__(self, maxsize, items) self._cond = Event() self._cond.set() if unfinished_tasks: self.unfinished_tasks = unfinished_tasks elif items: self.unfinished_tasks = len(items) else: self.unfinished_tasks = 0 if self.unfinished_tasks: self._cond.clear() def copy(self): return type(self)(self.maxsize, self.queue, self.unfinished_tasks) def _format(self): result = Queue._format(self) if self.unfinished_tasks: result += ' tasks=%s _cond=%s' % (self.unfinished_tasks, self._cond) return result def _put(self, item): Queue._put(self, item) self.unfinished_tasks += 1 self._cond.clear() def task_done(self): '''Indicate that a formerly enqueued task is complete. Used by queue consumer threads. For each :meth:`get <Queue.get>` used to fetch a task, a subsequent call to :meth:`task_done` tells the queue that the processing on the task is complete. If a :meth:`join` is currently blocking, it will resume when all items have been processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put <Queue.put>` into the queue). Raises a :exc:`ValueError` if called more times than there were items placed in the queue. ''' if self.unfinished_tasks <= 0: raise ValueError('task_done() called too many times') self.unfinished_tasks -= 1 if self.unfinished_tasks == 0: self._cond.set() def join(self, timeout=None): ''' Block until all items in the queue have been gotten and processed. The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer thread calls :meth:`task_done` to indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. :param float timeout: If not ``None``, then wait no more than this time in seconds for all tasks to finish. :return: ``True`` if all tasks have finished; if ``timeout`` was given and expired before all tasks finished, ``False``. .. versionchanged:: 1.1a1 Add the *timeout* parameter. ''' return self._cond.wait(timeout=timeout)
class TimerQueue(object): """A timer that provides efficient scheduling of large numbers of events in the near future.""" def __init__(self, time_source=time.time, resolution=0.01): """ Args: time_source - A callable to get the current time (in seconds). resolution - The minimum resolution of the timer. If set, all events are quantized to this resolution. """ self._queue = [] self._event = Event() self._seq = 0 self._resolution = resolution self._time_source = time_source self._worker = gevent.spawn(self._TimerWorker) def __del__(self): self._worker.kill(block=False) self._worker = None def _TimerWorker(self): while True: # If the queue is empty, wait for an item to be added. if not self._queue: self._event.wait() if self._event.is_set(): self._event.clear() # A sleep here is needed to work around a bug with gevent. # If the event is cleared and then immediately waited on, the wait will # completed instantly and report it timed out. gevent.sleep(0) # Peek the head of the queue at, peeked_seq, cancelled = self._PeekNext() if cancelled: # The item has already been canceled, remove it and continue. heapq.heappop(self._queue) continue # Wait for a new item to be added to the queue, # or for the timeout to expire. to_wait = at - self._time_source() if to_wait > 0: # There's some time to wait, wait for it or a more recent item to be # added to the head of the queue. wait_timed_out = not self._event.wait(to_wait) else: # It should already be run, do it right now. wait_timed_out = True if wait_timed_out: # Nothing newer came in before it timed out. at, seq, cancelled, action = heapq.heappop(self._queue) # This is an assert that should never occur, if it does, we somehow # ran events out of order. if seq != peeked_seq: LOG.critical("seq != peeked_seq [%d, %d]" % (seq, peeked_seq)) if not cancelled: # Run it gevent.spawn(action) # Clear the reference out in this loop del action else: # A newer item came in, nothing to do here, re-loop pass def _PeekNext(self): return self._queue[0][:3] def Schedule(self, deadline, action): """Schedule an operation Args: deadline - The absolute time this event should occur on. action - The action to run. Returns: A callable that can be invoked to cancel the scheduled operation. """ if action is None: raise Exception("action must be non-null") if self._resolution: deadline = int(math.ceil( float(deadline) / self._resolution)) * self._resolution self._seq += 1 timeout_args = [deadline, self._seq, False, action] def cancel(): timeout_args[2] = True # Null out to avoid holding onto references. timeout_args[3] = None heapq.heappush(self._queue, timeout_args) # Wake up the waiter thread if this is now the newest if self._queue[0][0] == deadline: self._event.set() return cancel
class UDPTransport(Runnable): UDP_MAX_MESSAGE_SIZE = 1200 def __init__(self, discovery, udpsocket, throttle_policy, config): super().__init__() # these values are initialized by the start method self.queueids_to_queues: typing.Dict self.raiden: RaidenService self.discovery = discovery self.config = config self.retry_interval = config['retry_interval'] self.retries_before_backoff = config['retries_before_backoff'] self.nat_keepalive_retries = config['nat_keepalive_retries'] self.nat_keepalive_timeout = config['nat_keepalive_timeout'] self.nat_invitation_timeout = config['nat_invitation_timeout'] self.event_stop = Event() self.event_stop.set() self.greenlets = list() self.addresses_events = dict() self.messageids_to_asyncresults = dict() # Maps the addresses to a dict with the latest nonce (using a dict # because python integers are immutable) self.nodeaddresses_to_nonces = dict() cache = cachetools.TTLCache( maxsize=50, ttl=CACHE_TTL, ) cache_wrapper = cachetools.cached(cache=cache) self.get_host_port = cache_wrapper(discovery.get) self.throttle_policy = throttle_policy self.server = DatagramServer(udpsocket, handle=self.receive) def start( self, raiden: RaidenService, ): if not self.event_stop.ready(): raise RuntimeError('UDPTransport started while running') self.event_stop.clear() self.raiden = raiden self.queueids_to_queues = dict() # server.stop() clears the handle. Since this may be a restart the # handle must always be set self.server.set_handle(self.receive) self.server.start() super().start() def _run(self): """ Runnable main method, perform wait on long-running subtasks """ try: self.event_stop.wait() except gevent.GreenletExit: # killed without exception self.event_stop.set() gevent.killall(self.greenlets) # kill children raise # re-raise to keep killed status except Exception: self.stop() # ensure cleanup and wait on subtasks raise def stop(self): if self.event_stop.ready(): return # double call, happens on normal stop, ignore self.event_stop.set() # Stop handling incoming packets, but don't close the socket. The # socket can only be safely closed after all outgoing tasks are stopped self.server.stop_accepting() # Stop processing the outgoing queues gevent.wait(self.greenlets) # All outgoing tasks are stopped. Now it's safe to close the socket. At # this point there might be some incoming message being processed, # keeping the socket open is not useful for these. self.server.stop() # Calling `.close()` on a gevent socket doesn't actually close the underlying os socket # so we do that ourselves here. # See: https://github.com/gevent/gevent/blob/master/src/gevent/_socket2.py#L208 # and: https://groups.google.com/forum/#!msg/gevent/Ro8lRra3nH0/ZENgEXrr6M0J try: self.server._socket.close() # pylint: disable=protected-access except socket.error: pass # Set all the pending results to False for async_result in self.messageids_to_asyncresults.values(): async_result.set(False) def get_health_events(self, recipient): """ Starts a healthcheck task for `recipient` and returns a HealthEvents with locks to react on its current state. """ if recipient not in self.addresses_events: self.start_health_check(recipient) return self.addresses_events[recipient] def start_health_check(self, recipient): """ Starts a task for healthchecking `recipient` if there is not one yet. """ if recipient not in self.addresses_events: ping_nonce = self.nodeaddresses_to_nonces.setdefault( recipient, {'nonce': 0}, # HACK: Allows the task to mutate the object ) events = healthcheck.HealthEvents( event_healthy=Event(), event_unhealthy=Event(), ) self.addresses_events[recipient] = events greenlet_healthcheck = gevent.spawn( healthcheck.healthcheck, self, recipient, self.event_stop, events.event_healthy, events.event_unhealthy, self.nat_keepalive_retries, self.nat_keepalive_timeout, self.nat_invitation_timeout, ping_nonce, ) greenlet_healthcheck.name = f'Healthcheck for {pex(recipient)}' greenlet_healthcheck.link_exception(self.on_error) self.greenlets.append(greenlet_healthcheck) def init_queue_for( self, queue_identifier: QueueIdentifier, items: typing.List[QueueItem_T], ) -> Queue_T: """ Create the queue identified by the queue_identifier and initialize it with `items`. """ recipient = queue_identifier.recipient queue = self.queueids_to_queues.get(queue_identifier) assert queue is None queue = NotifyingQueue(items=items) self.queueids_to_queues[queue_identifier] = queue events = self.get_health_events(recipient) greenlet_queue = gevent.spawn( single_queue_send, self, recipient, queue, queue_identifier, self.event_stop, events.event_healthy, events.event_unhealthy, self.retries_before_backoff, self.retry_interval, self.retry_interval * 10, ) if queue_identifier.channel_identifier == CHANNEL_IDENTIFIER_GLOBAL_QUEUE: greenlet_queue.name = f'Queue for {pex(recipient)} - global' else: greenlet_queue.name = ( f'Queue for {pex(recipient)} - {queue_identifier.channel_identifier}' ) greenlet_queue.link_exception(self.on_error) self.greenlets.append(greenlet_queue) log.debug( 'new queue created for', node=pex(self.raiden.address), queue_identifier=queue_identifier, items_qty=len(items), ) return queue def get_queue_for( self, queue_identifier: QueueIdentifier, ) -> Queue_T: """ Return the queue identified by the given queue identifier. If the queue doesn't exist it will be instantiated. """ queue = self.queueids_to_queues.get(queue_identifier) if queue is None: items = () queue = self.init_queue_for(queue_identifier, items) return queue def send_async( self, queue_identifier: QueueIdentifier, message: 'Message', ): """ Send a new ordered message to recipient. Messages that use the same `queue_identifier` are ordered. """ recipient = queue_identifier.recipient if not is_binary_address(recipient): raise ValueError('Invalid address {}'.format(pex(recipient))) # These are not protocol messages, but transport specific messages if isinstance(message, (Delivered, Ping, Pong)): raise ValueError('Do not use send for {} messages'.format(message.__class__.__name__)) messagedata = message.encode() if len(messagedata) > self.UDP_MAX_MESSAGE_SIZE: raise ValueError( 'message size exceeds the maximum {}'.format(self.UDP_MAX_MESSAGE_SIZE), ) # message identifiers must be unique message_id = message.message_identifier # ignore duplicates if message_id not in self.messageids_to_asyncresults: self.messageids_to_asyncresults[message_id] = AsyncResult() queue = self.get_queue_for(queue_identifier) queue.put((messagedata, message_id)) assert queue.is_set() log.debug( 'MESSAGE QUEUED', node=pex(self.raiden.address), queue_identifier=queue_identifier, queue_size=len(queue), message=message, ) def maybe_send(self, recipient: typing.Address, message: Message): """ Send message to recipient if the transport is running. """ if not is_binary_address(recipient): raise InvalidAddress('Invalid address {}'.format(pex(recipient))) messagedata = message.encode() host_port = self.get_host_port(recipient) self.maybe_sendraw(host_port, messagedata) def maybe_sendraw_with_result( self, recipient: typing.Address, messagedata: bytes, message_id: typing.MessageID, ) -> AsyncResult: """ Send message to recipient if the transport is running. Returns: An AsyncResult that will be set once the message is delivered. As long as the message has not been acknowledged with a Delivered message the function will return the same AsyncResult. """ async_result = self.messageids_to_asyncresults.get(message_id) if async_result is None: async_result = AsyncResult() self.messageids_to_asyncresults[message_id] = async_result host_port = self.get_host_port(recipient) self.maybe_sendraw(host_port, messagedata) return async_result def maybe_sendraw(self, host_port: typing.Tuple[int, int], messagedata: bytes): """ Send message to recipient if the transport is running. """ # Don't sleep if timeout is zero, otherwise a context-switch is done # and the message is delayed, increasing it's latency sleep_timeout = self.throttle_policy.consume(1) if sleep_timeout: gevent.sleep(sleep_timeout) # Check the udp socket is still available before trying to send the # message. There must be *no context-switches after this test*. if hasattr(self.server, 'socket'): self.server.sendto( messagedata, host_port, ) def receive( self, messagedata: bytes, host_port: typing.Tuple[str, int], # pylint: disable=unused-argument ) -> bool: """ Handle an UDP packet. """ # pylint: disable=unidiomatic-typecheck if len(messagedata) > self.UDP_MAX_MESSAGE_SIZE: log.warning( 'INVALID MESSAGE: Packet larger than maximum size', node=pex(self.raiden.address), message=hexlify(messagedata), length=len(messagedata), ) return False try: message = decode(messagedata) except InvalidProtocolMessage as e: log.warning( 'INVALID PROTOCOL MESSAGE', error=str(e), node=pex(self.raiden.address), message=hexlify(messagedata), ) return False if type(message) == Pong: self.receive_pong(message) elif type(message) == Ping: self.receive_ping(message) elif type(message) == Delivered: self.receive_delivered(message) elif message is not None: self.receive_message(message) else: log.warning( 'INVALID MESSAGE: Unknown cmdid', node=pex(self.raiden.address), message=hexlify(messagedata), ) return False return True def receive_message(self, message: Message): """ Handle a Raiden protocol message. The protocol requires durability of the messages. The UDP transport relies on the node's WAL for durability. The message will be converted to a state change, saved to the WAL, and *processed* before the durability is confirmed, which is a stronger property than what is required of any transport. """ # pylint: disable=unidiomatic-typecheck if on_message(self.raiden, message): # Sending Delivered after the message is decoded and *processed* # gives a stronger guarantee than what is required from a # transport. # # Alternatives are, from weakest to strongest options: # - Just save it on disk and asynchronously process the messages # - Decode it, save to the WAL, and asynchronously process the # state change # - Decode it, save to the WAL, and process it (the current # implementation) delivered_message = Delivered(message.message_identifier) self.raiden.sign(delivered_message) self.maybe_send( message.sender, delivered_message, ) def receive_delivered(self, delivered: Delivered): """ Handle a Delivered message. The Delivered message is how the UDP transport guarantees persistence by the partner node. The message itself is not part of the raiden protocol, but it's required by this transport to provide the required properties. """ processed = ReceiveDelivered(delivered.delivered_message_identifier) self.raiden.handle_state_change(processed) message_id = delivered.delivered_message_identifier async_result = self.raiden.transport.messageids_to_asyncresults.get(message_id) # clear the async result, otherwise we have a memory leak if async_result is not None: del self.messageids_to_asyncresults[message_id] async_result.set() else: log.warn( 'UNKNOWN DELIVERED MESSAGE RECEIVED', message_id=message_id, ) # Pings and Pongs are used to check the health status of another node. They # are /not/ part of the raiden protocol, only part of the UDP transport, # therefore these messages are not forwarded to the message handler. def receive_ping(self, ping: Ping): """ Handle a Ping message by answering with a Pong. """ log_healthcheck.debug( 'PING RECEIVED', node=pex(self.raiden.address), message_id=ping.nonce, message=ping, sender=pex(ping.sender), ) pong = Pong(ping.nonce) self.raiden.sign(pong) try: self.maybe_send(ping.sender, pong) except (InvalidAddress, UnknownAddress) as e: log.debug("Couldn't send the `Delivered` message", e=e) def receive_pong(self, pong: Pong): """ Handles a Pong message. """ message_id = ('ping', pong.nonce, pong.sender) async_result = self.messageids_to_asyncresults.get(message_id) if async_result is not None: log_healthcheck.debug( 'PONG RECEIVED', node=pex(self.raiden.address), sender=pex(pong.sender), message_id=pong.nonce, ) async_result.set(True) else: log_healthcheck.warn( 'UNKNOWN PONG RECEIVED', message_id=message_id, ) def get_ping(self, nonce: int) -> Ping: """ Returns a signed Ping message. Note: Ping messages don't have an enforced ordering, so a Ping message with a higher nonce may be acknowledged first. """ message = Ping(nonce) self.raiden.sign(message) message_data = message.encode() return message_data def set_node_network_state(self, node_address: typing.Address, node_state): state_change = ActionChangeNodeNetworkState(node_address, node_state) self.raiden.handle_state_change(state_change)
class SteamUser(object): """ A data model for a Steam user. Holds user persona state, and related actions .. note:: This is an internal object that can be obtained by :meth:`SteamClient.get_user` """ _pstate = None _pstate_requested = False steam_id = SteamID() #: steam id relationship = EFriendRelationship.NONE #: friendship status def __init__(self, steam_id, steam): self._pstate_ready = Event() self._steam = steam self.steam_id = SteamID(steam_id) def __repr__(self): return "<%s(%s, %s, %s)>" % ( self.__class__.__name__, str(self.steam_id), self.relationship, self.state, ) def refresh(self, wait=True): if self._pstate_requested and self._pstate_ready.is_set(): self._pstate_requested = False if not self._pstate_requested: self._steam.request_persona_state([self.steam_id]) self._pstate_ready.clear() self._pstate_requested = True if wait: self._pstate_ready.wait(timeout=5) self._pstate_requested = False def get_ps(self, field_name, wait_pstate=True): """Get property from PersonaState `See full list of available fields_names <https://github.com/ValvePython/steam/blob/fa8a5127e9bb23185483930da0b6ae85e93055a7/protobufs/steammessages_clientserver_friends.proto#L125-L153>`_ """ if not self._pstate_ready.is_set() and wait_pstate: self.refresh() if self._pstate is not None: return getattr(self._pstate, field_name) else: return None @property def last_logon(self): """:rtype: :class:`datetime`, :class:`None`""" ts = self.get_ps('last_logon') return datetime.utcfromtimestamp(ts) if ts else None @property def last_logoff(self): """:rtype: :class:`datetime`, :class:`None`""" ts = self.get_ps('last_logoff') return datetime.utcfromtimestamp(ts) if ts else None @property def name(self): """Name of the steam user, or ``None`` if it's not available :rtype: :class:`str`, :class:`None` """ return self.get_ps('player_name') @property def state(self): """Personsa state (e.g. Online, Offline, Away, Busy, etc) :rtype: :class:`.EPersonaState` """ state = self.get_ps('persona_state', False) return EPersonaState(state) if state else EPersonaState.Offline @property def rich_presence(self): """Contains Rich Presence key-values :rtype: dict """ kvs = self.get_ps('rich_presence') data = {} if kvs: for kv in kvs: data[kv.key] = kv.value return data def get_avatar_url(self, size=2): """Get URL to avatar picture :param size: possible values are ``0``, ``1``, or ``2`` corresponding to small, medium, large :type size: :class:`int` :return: url to avatar :rtype: :class:`str` """ hashbytes = self.get_ps('avatar_hash') if hashbytes != "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000": ahash = hexlify(hashbytes).decode('ascii') else: ahash = 'fef49e7fa7e1997310d705b2a6158ff8dc1cdfeb' sizes = { 0: '', 1: '_medium', 2: '_full', } url = "http://cdn.akamai.steamstatic.com/steamcommunity/public/images/avatars/%s/%s%s.jpg" return url % (ahash[:2], ahash, sizes[size]) def send_message(self, message): """Send chat message to this steam user :param message: message to send :type message: str """ # new chat if self._steam.chat_mode == 2: self._steam.send_um("FriendMessages.SendMessage#1", { 'steamid': self.steam_id, 'message': message, 'chat_entry_type': EChatEntryType.ChatMsg, }) # old chat else: self._steam.send(MsgProto(EMsg.ClientFriendMsg), { 'steamid': self.steam_id, 'chat_entry_type': EChatEntryType.ChatMsg, 'message': message.encode('utf8'), }) def block(self): """Block user""" self._steam.friends.block(self) def unblock(self): """Unblock user""" self._steam.friends.unblock(self)
def main(): stop_event = Event() parser = argparse.ArgumentParser() parser.add_argument( '--mock-networking', action='store_true', help='In-Process Trader, MessageBroker and CommitmentService') parser.add_argument( '--mock', action='store_true', help='Spawns mock offers to simulate trading activity"') parser.add_argument('--seed', type=str, default='raidex-node', help='Use the keccak privkey from seed') parser.add_argument('--keyfile', type=argparse.FileType('r'), help='path to keyfile') parser.add_argument('--pwfile', type=argparse.FileType('r'), help='path to pw') parser.add_argument("--api", action='store_true', help='Run the REST-API') parser.add_argument("--api-port", type=int, help='Specify the port for the api, default is 50001', default=50001) parser.add_argument("--offer-lifetime", type=int, help='Lifetime of offers spawned by LimitOrders', default=30) parser.add_argument( "--broker-host", type=str, help='Specify the host for the message broker, default is localhost', default='localhost') parser.add_argument( "--broker-port", type=int, help='Specify the port for the message broker, default is 5000', default=5000) parser.add_argument( "--trader-host", type=str, help='Specify the host for the trader mock, default is localhost', default='localhost') parser.add_argument( "--trader-port", type=int, help='Specify the port for the trader mock, default is 5001', default=5001) parser.add_argument('--bots', nargs='+', help='Start a set of (/subset of) multiple tradi' 'ng bots.\ <Options:\"liquidity\", \"random\", \"manipulator\">' ) parser.add_argument( '--token-address', type=str, help='Token address of token to trade against WETH on kovan', default='0x92276aD441CA1F3d8942d614a6c3c87592dd30bb') args = parser.parse_args() if args.mock_networking is True: message_broker = MessageBroker() commitment_service = CommitmentService.build_service(message_broker, fee_rate=1) raidex_app = App.build_from_mocks(message_broker, commitment_service.address, base_token_addr=args.token_address, quote_token_addr=KOVAN_WETH_ADDRESS, keyfile=args.keyfile, pw_file=args.pwfile, offer_lifetime=args.offer_lifetime) commitment_service.start() else: raidex_app = App.build_default_from_config( keyfile=args.keyfile, pw_file=args.pwfile, cs_address=CS_ADDRESS, base_token_addr=args.token_address, quote_token_addr=KOVAN_WETH_ADDRESS, message_broker_host=args.broker_host, message_broker_port=args.broker_port, trader_host=args.trader_host, trader_port=args.trader_port, mock_trading_activity=args.mock, offer_lifetime=args.offer_lifetime) raidex_app.start() if args.api is True: api = APIServer('', args.api_port, raidex_app.raidex_node) api.start() bots = args.bots if bots: initial_price = 100. if 'liquidity' in bots: liquidity_provider = LiquidityProvider(raidex_app, initial_price) liquidity_provider.start() if 'random' in bots: gevent.sleep(5) # give liquidity provider head start random_walker = RandomWalker(raidex_app, initial_price) random_walker.start() if 'maniplulator' in bots: if 'random' not in bots: gevent.sleep(5) # give liquidity provider head start manipulator = Manipulator(raidex_app, initial_price) manipulator.start() stop_event.wait() # runs forever
class ReplayProcess(BaseReplayProcess): ''' ReplayProcess - A process spawned for the purpose of replaying data -------------------------------------------------------------------------------- Configurations ============== process: dataset_id: "" # Dataset to be replayed delivery_format: {} # Delivery format to be replayed back (unused for now) query: start_time: 0 # Start time (index value) to be replayed end_time: 0 # End time (index value) to be replayed parameters: [] # List of parameters to form in the granule ''' process_type = 'standalone' publish_limit = 10 dataset_id = None delivery_format = {} start_time = None end_time = None stride_time = None parameters = None stream_id = '' stream_def_id = '' def __init__(self, *args, **kwargs): super(ReplayProcess,self).__init__(*args,**kwargs) self.deserializer = IonObjectDeserializer(obj_registry=get_obj_registry()) self.publishing = Event() self.play = Event() self.end = Event() def on_start(self): ''' Starts the process ''' log.info('Replay Process Started') super(ReplayProcess,self).on_start() dsm_cli = DatasetManagementServiceProcessClient(process=self) pubsub = PubsubManagementServiceProcessClient(process=self) self.dataset_id = self.CFG.get_safe('process.dataset_id', None) self.delivery_format = self.CFG.get_safe('process.delivery_format',{}) self.start_time = self.CFG.get_safe('process.query.start_time', None) self.end_time = self.CFG.get_safe('process.query.end_time', None) self.stride_time = self.CFG.get_safe('process.query.stride_time', None) self.parameters = self.CFG.get_safe('process.query.parameters',None) self.publish_limit = self.CFG.get_safe('process.query.publish_limit', 10) self.tdoa = self.CFG.get_safe('process.query.tdoa',None) self.stream_id = self.CFG.get_safe('process.publish_streams.output', '') self.stream_def = pubsub.read_stream_definition(stream_id=self.stream_id) self.stream_def_id = self.stream_def._id self.replay_thread = None self.publishing.clear() self.play.set() self.end.clear() if self.dataset_id is None: raise BadRequest('dataset_id not specified') self.dataset = dsm_cli.read_dataset(self.dataset_id) self.pubsub = PubsubManagementServiceProcessClient(process=self) @classmethod def get_time_idx(cls, coverage, timeval): corrected_time = cls.convert_time(coverage, timeval) idx = TimeUtils.get_relative_time(coverage, corrected_time) return idx @classmethod def convert_time(cls, coverage, timeval): tname = coverage.temporal_parameter_name uom = coverage.get_parameter_context(tname).uom corrected_time = TimeUtils.ts_to_units(uom, timeval) return corrected_time @classmethod def _data_dict_to_rdt(cls, data_dict, stream_def_id, coverage): if stream_def_id: rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) else: rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary) if not data_dict: log.warning('Retrieve returning empty set') return rdt if 'time' in data_dict and data_dict['time'].shape[0] == 0: log.warning('Retrieve returning empty set') return rdt rdt[coverage.temporal_parameter_name] = data_dict[coverage.temporal_parameter_name] for field in rdt.fields: if field == coverage.temporal_parameter_name: continue # The values have already been inside a coverage so we know they're safe and they exist, so they can be inserted directly. if field in data_dict: rdt._rd[field] = data_dict[field] #rdt[k] = v return rdt @classmethod def _cov2granule(cls, coverage, start_time=None, end_time=None, stride_time=None, stream_def_id=None, parameters=None, tdoa=None, sort_parameter=None): # Deal with the NTP if start_time: start_time += 2208988800 if end_time: end_time += 2208988800 if tdoa is None: if start_time is None and end_time is None: data_dict = coverage.get_parameter_values(param_names=parameters, stride_length=stride_time, fill_empty_params=True, sort_parameter=sort_parameter).get_data() else: data_dict = coverage.get_parameter_values(param_names=parameters, time_segment=(start_time, end_time), stride_length=stride_time, fill_empty_params=True, sort_parameter=sort_parameter).get_data() elif isinstance(tdoa, slice): log.warning("Using tdoa argument on large datasets can consume too much memory") data_dict = coverage.get_parameter_values(param_names=parameters, fill_empty_params=True).get_data() data_dict = { k : v[tdoa] for k,v in data_dict.iteritems() } else: raise TypeError("tdoa is incorrect type: %s" % type(tdoa)) return cls._data_dict_to_rdt(data_dict, stream_def_id, coverage) def execute_retrieve(self): ''' execute_retrieve Executes a retrieval and returns the result as a value in lieu of publishing it on a stream ''' try: coverage = DatasetManagementService._get_coverage(self.dataset_id,mode='r') if coverage.is_empty(): log.info('Reading from an empty coverage') rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary) else: rdt = ReplayProcess._cov2granule(coverage=coverage, start_time=self.start_time, end_time=self.end_time, stride_time=self.stride_time, parameters=self.parameters, stream_def_id=self.delivery_format, tdoa=self.tdoa) except: log.exception('Problems reading from the coverage') raise BadRequest('Problems reading from the coverage') finally: coverage.close(timeout=5) return rdt.to_granule() @classmethod def get_last_values(cls, dataset_id, number_of_points=100, delivery_format=''): stream_def_id = delivery_format try: cov = DatasetManagementService._get_coverage(dataset_id, mode='r') if cov.is_empty(): rdt = RecordDictionaryTool(param_dictionary=cov.parameter_dictionary) else: time_array = cov.get_parameter_values([cov.temporal_parameter_name], sort_parameter=cov.temporal_parameter_name).get_data() time_array = time_array[cov.temporal_parameter_name][-number_of_points:] t0 = np.asscalar(time_array[0]) t1 = np.asscalar(time_array[-1]) data_dict = cov.get_parameter_values(time_segment=(t0, t1), fill_empty_params=True).get_data() rdt = cls._data_dict_to_rdt(data_dict, stream_def_id, cov) except: log.exception('Problems reading from the coverage') raise BadRequest('Problems reading from the coverage') finally: if cov is not None: cov.close(timeout=5) return rdt def execute_replay(self): ''' execute_replay Performs a replay and publishes the results on a stream. ''' if self.publishing.is_set(): return False self.replay_thread = self._process.thread_manager.spawn(self.replay) return True def replay(self): self.publishing.set() # Minimal state, supposed to prevent two instances of the same process from replaying on the same stream for rdt in self._replay(): if self.end.is_set(): return self.play.wait() self.output.publish(rdt.to_granule()) self.publishing.clear() return def pause(self): self.play.clear() def resume(self): self.play.set() def stop(self): self.end.set() def _replay(self): coverage = DatasetManagementService._get_coverage(self.dataset_id,mode='r') rdt = self._cov2granule(coverage=coverage, start_time=self.start_time, end_time=self.end_time, stride_time=self.stride_time, parameters=self.parameters, stream_def_id=self.stream_def_id) elements = len(rdt) for i in xrange(elements / self.publish_limit): outgoing = RecordDictionaryTool(stream_definition_id=self.stream_def_id) fields = self.parameters or outgoing.fields for field in fields: v = rdt[field] if v is not None: outgoing[field] = v[(i*self.publish_limit) : ((i+1)*self.publish_limit)] yield outgoing coverage.close(timeout=5) return
class Group(GroupMappingMixin): """ Maintain a group of greenlets that are still running, without limiting their number. Links to each item and removes it upon notification. Groups can be iterated to discover what greenlets they are tracking, they can be tested to see if they contain a greenlet, and they know the number (len) of greenlets they are tracking. If they are not tracking any greenlets, they are False in a boolean context. .. attribute:: greenlet_class Either :class:`gevent.Greenlet` (the default) or a subclass. These are the type of object we will :meth:`spawn`. This can be changed on an instance or in a subclass. """ greenlet_class = Greenlet def __init__(self, *args): assert len(args) <= 1, args self.greenlets = set(*args) if args: for greenlet in args[0]: greenlet.rawlink(self._discard) # each item we kill we place in dying, to avoid killing the same greenlet twice self.dying = set() self._empty_event = Event() self._empty_event.set() def __repr__(self): return '<%s at 0x%x %s>' % (self.__class__.__name__, id(self), self.greenlets) def __len__(self): """ Answer how many greenlets we are tracking. Note that if we are empty, we are False in a boolean context. """ return len(self.greenlets) def __contains__(self, item): """ Answer if we are tracking the given greenlet. """ return item in self.greenlets def __iter__(self): """ Iterate across all the greenlets we are tracking, in no particular order. """ return iter(self.greenlets) def add(self, greenlet): """ Begin tracking the *greenlet*. If this group is :meth:`full`, then this method may block until it is possible to track the greenlet. Typically the *greenlet* should **not** be started when it is added because if this object blocks in this method, then the *greenlet* may run to completion before it is tracked. """ try: rawlink = greenlet.rawlink except AttributeError: pass # non-Greenlet greenlet, like MAIN else: rawlink(self._discard) self.greenlets.add(greenlet) self._empty_event.clear() def _discard(self, greenlet): self.greenlets.discard(greenlet) self.dying.discard(greenlet) if not self.greenlets: self._empty_event.set() def discard(self, greenlet): """ Stop tracking the greenlet. """ self._discard(greenlet) try: unlink = greenlet.unlink except AttributeError: pass # non-Greenlet greenlet, like MAIN else: unlink(self._discard) def start(self, greenlet): """ Add the **unstarted** *greenlet* to the collection of greenlets this group is monitoring, and then start it. """ self.add(greenlet) greenlet.start() def spawn(self, *args, **kwargs): # pylint:disable=arguments-differ """ Begin a new greenlet with the given arguments (which are passed to the greenlet constructor) and add it to the collection of greenlets this group is monitoring. :return: The newly started greenlet. """ greenlet = self.greenlet_class(*args, **kwargs) self.start(greenlet) return greenlet # def close(self): # """Prevents any more tasks from being submitted to the pool""" # self.add = RaiseException("This %s has been closed" % self.__class__.__name__) def join(self, timeout=None, raise_error=False): """ Wait for this group to become empty *at least once*. If there are no greenlets in the group, returns immediately. .. note:: By the time the waiting code (the caller of this method) regains control, a greenlet may have been added to this group, and so this object may no longer be empty. (That is, ``group.join(); assert len(group) == 0`` is not guaranteed to hold.) This method only guarantees that the group reached a ``len`` of 0 at some point. :keyword bool raise_error: If True (*not* the default), if any greenlet that finished while the join was in progress raised an exception, that exception will be raised to the caller of this method. If multiple greenlets raised exceptions, which one gets re-raised is not determined. Only greenlets currently in the group when this method is called are guaranteed to be checked for exceptions. :return bool: A value indicating whether this group became empty. If the timeout is specified and the group did not become empty during that timeout, then this will be a false value. Otherwise it will be a true value. .. versionchanged:: 1.2a1 Add the return value. """ greenlets = list(self.greenlets) if raise_error else () result = self._empty_event.wait(timeout=timeout) for greenlet in greenlets: if greenlet.exception is not None: if hasattr(greenlet, '_raise_exception'): greenlet._raise_exception() raise greenlet.exception return result def kill(self, exception=GreenletExit, block=True, timeout=None): """ Kill all greenlets being tracked by this group. """ timer = Timeout._start_new_or_dummy(timeout) try: while self.greenlets: for greenlet in list(self.greenlets): if greenlet in self.dying: continue try: kill = greenlet.kill except AttributeError: _kill(greenlet, exception) else: kill(exception, block=False) self.dying.add(greenlet) if not block: break joinall(self.greenlets) except Timeout as ex: if ex is not timer: raise finally: timer.cancel() def killone(self, greenlet, exception=GreenletExit, block=True, timeout=None): """ If the given *greenlet* is running and being tracked by this group, kill it. """ if greenlet not in self.dying and greenlet in self.greenlets: greenlet.kill(exception, block=False) self.dying.add(greenlet) if block: greenlet.join(timeout) def full(self): """ Return a value indicating whether this group can track more greenlets. In this implementation, because there are no limits on the number of tracked greenlets, this will always return a ``False`` value. """ return False def wait_available(self, timeout=None): """ Block until it is possible to :meth:`spawn` a new greenlet. In this implementation, because there are no limits on the number of tracked greenlets, this will always return immediately. """ # MappingMixin methods def _apply_immediately(self): # If apply() is called from one of our own # worker greenlets, don't spawn a new one---if we're full, that # could deadlock. return getcurrent() in self def _apply_async_cb_spawn(self, callback, result): Greenlet.spawn(callback, result) def _apply_async_use_greenlet(self): # cannot call self.spawn() because it will block, so # use a fresh, untracked greenlet that when run will # (indirectly) call self.spawn() for us. return self.full()
class TestDMEnd2End(IonIntegrationTestCase): def setUp(self): # Love the non pep-8 convention self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.process_dispatcher = ProcessDispatcherServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.ingestion_management = IngestionManagementServiceClient() self.data_retriever = DataRetrieverServiceClient() self.pids = [] self.event = Event() self.exchange_space_name = 'test_granules' self.exchange_point_name = 'science_data' self.i = 0 self.purge_queues() self.queue_buffer = [] self.streams = [] self.addCleanup(self.stop_all_ingestion) def purge_queues(self): xn = self.container.ex_manager.create_xn_queue( 'science_granule_ingestion') xn.purge() def tearDown(self): self.purge_queues() for pid in self.pids: self.container.proc_manager.terminate_process(pid) IngestionManagementIntTest.clean_subscriptions() for queue in self.queue_buffer: if isinstance(queue, ExchangeNameQueue): queue.delete() elif isinstance(queue, str): xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() #-------------------------------------------------------------------------------- # Helper/Utility methods #-------------------------------------------------------------------------------- def create_dataset(self, parameter_dict_id=''): ''' Creates a time-series dataset ''' tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() if not parameter_dict_id: parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) dataset_id = self.dataset_management.create_dataset( 'test_dataset_%i' % self.i, parameter_dictionary_id=parameter_dict_id, spatial_domain=sdom, temporal_domain=tdom) return dataset_id def get_datastore(self, dataset_id): ''' Gets an instance of the datastore This method is primarily used to defeat a bug where integration tests in multiple containers may sometimes delete a CouchDB datastore and the other containers are unaware of the new state of the datastore. ''' dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def get_ingestion_config(self): ''' Grab the ingestion configuration from the resource registry ''' # The ingestion configuration should have been created by the bootstrap service # which is configured through r2deploy.yml ingest_configs, _ = self.resource_registry.find_resources( restype=RT.IngestionConfiguration, id_only=True) return ingest_configs[0] def launch_producer(self, stream_id=''): ''' Launch the producer ''' pid = self.container.spawn_process( 'better_data_producer', 'ion.processes.data.example_data_producer', 'BetterDataProducer', {'process': { 'stream_id': stream_id }}) self.pids.append(pid) def make_simple_dataset(self): ''' Makes a stream, a stream definition and a dataset, the essentials for most of these tests ''' pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition( 'ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream( 'ctd stream %i' % self.i, 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) self.get_datastore(dataset_id) self.i += 1 return stream_id, route, stream_def_id, dataset_id def publish_hifi(self, stream_id, stream_route, offset=0): ''' Publish deterministic data ''' pub = StandaloneStreamPublisher(stream_id, stream_route) stream_def = self.pubsub_management.read_stream_definition( stream_id=stream_id) stream_def_id = stream_def._id rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) + (offset * 10) rdt['temp'] = np.arange(10) + (offset * 10) pub.publish(rdt.to_granule()) def publish_fake_data(self, stream_id, route): ''' Make four granules ''' for i in xrange(4): self.publish_hifi(stream_id, route, i) def start_ingestion(self, stream_id, dataset_id): ''' Starts ingestion/persistence for a given dataset ''' ingest_config_id = self.get_ingestion_config() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) def stop_ingestion(self, stream_id): ingest_config_id = self.get_ingestion_config() self.ingestion_management.unpersist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingest_config_id) def stop_all_ingestion(self): try: [self.stop_ingestion(sid) for sid in self.streams] except: pass def validate_granule_subscription(self, msg, route, stream_id): ''' Validation for granule format ''' if msg == {}: return rdt = RecordDictionaryTool.load_from_granule(msg) log.info('%s', rdt.pretty_print()) self.assertIsInstance( msg, Granule, 'Message is improperly formatted. (%s)' % type(msg)) self.event.set() def wait_until_we_have_enough_granules(self, dataset_id='', data_size=40): ''' Loops until there is a sufficient amount of data in the dataset ''' done = False with gevent.Timeout(40): while not done: extents = self.dataset_management.dataset_extents( dataset_id, 'time')[0] granule = self.data_retriever.retrieve_last_data_points( dataset_id, 1) rdt = RecordDictionaryTool.load_from_granule(granule) if rdt['time'] and rdt['time'][0] != rdt._pdict.get_context( 'time').fill_value and extents >= data_size: done = True else: gevent.sleep(0.2) #-------------------------------------------------------------------------------- # Test Methods #-------------------------------------------------------------------------------- @attr('SMOKE') def test_dm_end_2_end(self): #-------------------------------------------------------------------------------- # Set up a stream and have a mock instrument (producer) send data #-------------------------------------------------------------------------------- self.event.clear() # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append( self.dataset_management.create_parameter_context( 'binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append( self.dataset_management.create_parameter_context( 'records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_definition = self.pubsub_management.create_stream_definition( 'ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream( 'producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) #-------------------------------------------------------------------------------- # Start persisting the data on the stream # - Get the ingestion configuration from the resource registry # - Create the dataset # - call persist_data_stream to setup the subscription for the ingestion workers # on the stream that you specify which causes the data to be persisted #-------------------------------------------------------------------------------- ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) #-------------------------------------------------------------------------------- # Now the granules are ingesting and persisted #-------------------------------------------------------------------------------- self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id, 40) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_id) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) self.assertTrue((rdt['time'][:10] == np.arange(10)).all(), '%s' % rdt['time'][:]) self.assertTrue((rdt['binary'][:10] == np.array(['hi'] * 10, dtype='object')).all()) #-------------------------------------------------------------------------------- # Now to try the streamed approach #-------------------------------------------------------------------------------- replay_stream_id, replay_route = self.pubsub_management.create_stream( 'replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) self.replay_id, process_id = self.data_retriever.define_replay( dataset_id=dataset_id, stream_id=replay_stream_id) log.info('Process ID: %s', process_id) replay_client = ReplayClient(process_id) #-------------------------------------------------------------------------------- # Create the listening endpoint for the the retriever to talk to #-------------------------------------------------------------------------------- xp = self.container.ex_manager.create_xp(self.exchange_point_name) subscriber = StandaloneStreamSubscriber( self.exchange_space_name, self.validate_granule_subscription) self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) self.data_retriever.start_replay_agent(self.replay_id) self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched') replay_client.start_replay() self.assertTrue(self.event.wait(10)) subscriber.stop() self.data_retriever.cancel_replay_agent(self.replay_id) #-------------------------------------------------------------------------------- # Test the slicing capabilities #-------------------------------------------------------------------------------- granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa': slice(0, 5)}) rdt = RecordDictionaryTool.load_from_granule(granule) b = rdt['time'] == np.arange(5) self.assertTrue(b.all() if not isinstance(b, bool) else b) self.streams.append(stream_id) self.stop_ingestion(stream_id) @unittest.skip('Doesnt work') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_replay_pause(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append( self.dataset_management.create_parameter_context( 'binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append( self.dataset_management.create_parameter_context( 'records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition( 'replay_stream', parameter_dictionary_id=pdict_id) replay_stream, replay_route = self.pubsub_management.create_stream( 'replay', 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) scov = DatasetManagementService._get_coverage(dataset_id) bb = CoverageCraft(scov) bb.rdt['time'] = np.arange(100) bb.rdt['temp'] = np.random.random(100) + 30 bb.sync_with_granule() DatasetManagementService._persist_coverage( dataset_id, bb.coverage) # This invalidates it for multi-host configurations # Set up the subscriber to verify the data subscriber = StandaloneStreamSubscriber( self.exchange_space_name, self.validate_granule_subscription) xp = self.container.ex_manager.create_xp('xp1') self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) # Set up the replay agent and the client wrapper # 1) Define the Replay (dataset and stream to publish on) self.replay_id, process_id = self.data_retriever.define_replay( dataset_id=dataset_id, stream_id=replay_stream) # 2) Make a client to the interact with the process (optionall provide it a process to bind with) replay_client = ReplayClient(process_id) # 3) Start the agent (launch the process) self.data_retriever.start_replay_agent(self.replay_id) # 4) Start replaying... replay_client.start_replay() # Wait till we get some granules self.assertTrue(self.event.wait(5)) # We got granules, pause the replay, clear the queue and allow the process to finish consuming replay_client.pause_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure there's no remaining messages being consumed self.assertFalse(self.event.wait(1)) # Resume the replay and wait until we start getting granules again replay_client.resume_replay() self.assertTrue(self.event.wait(5)) # Stop the replay, clear the queues replay_client.stop_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure that it did indeed stop self.assertFalse(self.event.wait(1)) subscriber.stop() def test_retrieve_and_transform(self): # Make a simple dataset and start ingestion, pretty standard stuff. ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) self.start_ingestion(ctd_stream_id, dataset_id) # Stream definition for the salinity data salinity_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) sal_stream_def_id = self.pubsub_management.create_stream_definition( 'sal data', parameter_dictionary_id=salinity_pdict_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) rdt['temp'] = np.random.randn(10) * 10 + 30 rdt['conductivity'] = np.random.randn(10) * 2 + 10 rdt['pressure'] = np.random.randn(10) * 1 + 12 publisher = StandaloneStreamPublisher(ctd_stream_id, route) publisher.publish(rdt.to_granule()) rdt['time'] = np.arange(10, 20) publisher.publish(rdt.to_granule()) self.wait_until_we_have_enough_granules(dataset_id, 20) granule = self.data_retriever.retrieve( dataset_id, None, None, 'ion.processes.data.transforms.ctd.ctd_L2_salinity', 'CTDL2SalinityTransformAlgorithm', kwargs=dict(params=sal_stream_def_id)) rdt = RecordDictionaryTool.load_from_granule(granule) for i in rdt['salinity']: self.assertNotEquals(i, 0) self.streams.append(ctd_stream_id) self.stop_ingestion(ctd_stream_id) def test_last_granule(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) self.start_ingestion(stream_id, dataset_id) self.publish_hifi(stream_id, route, 0) self.publish_hifi(stream_id, route, 1) self.wait_until_we_have_enough_granules(dataset_id, 20) # I just need two success = False def verifier(): replay_granule = self.data_retriever.retrieve_last_data_points( dataset_id, 10) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(10) + 10 if not isinstance(comp, bool): return comp.all() return False success = poll(verifier) self.assertTrue(success) success = False def verify_points(): replay_granule = self.data_retriever.retrieve_last_data_points( dataset_id, 5) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(15, 20) if not isinstance(comp, bool): return comp.all() return False success = poll(verify_points) self.assertTrue(success) self.streams.append(stream_id) self.stop_ingestion(stream_id) def test_replay_with_parameters(self): #-------------------------------------------------------------------------------- # Create the configurations and the dataset #-------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append( self.dataset_management.create_parameter_context( 'binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append( self.dataset_management.create_parameter_context( 'records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition( 'replay_stream', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream( 'replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id) es.start() self.addCleanup(es.stop) self.publish_fake_data(stream_id, route) self.assertTrue(dataset_modified.wait(30)) query = { 'start_time': 0 - 2208988800, 'end_time': 20 - 2208988800, 'stride_time': 2, 'parameters': ['time', 'temp'] } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id, query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(0, 20, 2) == rdt['time'] self.assertTrue(comp.all(), '%s' % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(['time', 'temp'])) extents = self.dataset_management.dataset_extents( dataset_id=dataset_id, parameters=['time', 'temp']) self.assertTrue(extents['time'] >= 20) self.assertTrue(extents['temp'] >= 20) self.streams.append(stream_id) self.stop_ingestion(stream_id) def test_repersist_data(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) self.start_ingestion(stream_id, dataset_id) self.publish_hifi(stream_id, route, 0) self.publish_hifi(stream_id, route, 1) self.wait_until_we_have_enough_granules(dataset_id, 20) config_id = self.get_ingestion_config() self.ingestion_management.unpersist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) self.publish_hifi(stream_id, route, 2) self.publish_hifi(stream_id, route, 3) self.wait_until_we_have_enough_granules(dataset_id, 40) success = False with gevent.timeout.Timeout(5): while not success: replay_granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(0, 40) if not isinstance(comp, bool): success = comp.all() gevent.sleep(1) self.assertTrue(success) self.streams.append(stream_id) self.stop_ingestion(stream_id) @attr('LOCOINT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Host requires file-system access to coverage files, CEI mode does not support.' ) def test_correct_time(self): # There are 2208988800 seconds between Jan 1 1900 and Jan 1 1970, i.e. # the conversion factor between unix and NTP time unix_now = np.floor(time.time()) ntp_now = unix_now + 2208988800 unix_ago = unix_now - 20 ntp_ago = unix_ago + 2208988800 stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) coverage = DatasetManagementService._get_coverage(dataset_id) coverage.insert_timesteps(20) coverage.set_parameter_values('time', np.arange(ntp_ago, ntp_now)) temporal_bounds = self.dataset_management.dataset_temporal_bounds( dataset_id) self.assertTrue(np.abs(temporal_bounds[0] - unix_ago) < 2) self.assertTrue(np.abs(temporal_bounds[1] - unix_now) < 2) @attr('LOCOINT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Host requires file-system access to coverage files, CEI mode does not support.' ) def test_empty_coverage_time(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) coverage = DatasetManagementService._get_coverage(dataset_id) temporal_bounds = self.dataset_management.dataset_temporal_bounds( dataset_id) self.assertEquals([coverage.get_parameter_context('time').fill_value] * 2, temporal_bounds) @attr('LOCOINT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Host requires file-system access to coverage files, CEI mode does not support.' ) def test_out_of_band_retrieve(self): # Setup the environemnt stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) self.start_ingestion(stream_id, dataset_id) # Fill the dataset self.publish_fake_data(stream_id, route) self.wait_until_we_have_enough_granules(dataset_id, 40) # Retrieve the data granule = DataRetrieverService.retrieve_oob(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) self.assertTrue((rdt['time'] == np.arange(40)).all()) @attr('LOCOINT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Host requires file-system access to coverage files, CEI mode does not support.' ) def test_retrieve_cache(self): DataRetrieverService._refresh_interval = 1 datasets = [self.make_simple_dataset() for i in xrange(10)] for stream_id, route, stream_def_id, dataset_id in datasets: coverage = DatasetManagementService._get_coverage(dataset_id) coverage.insert_timesteps(10) coverage.set_parameter_values('time', np.arange(10)) coverage.set_parameter_values('temp', np.arange(10)) # Verify cache hit and refresh dataset_ids = [i[3] for i in datasets] self.assertTrue( dataset_ids[0] not in DataRetrieverService._retrieve_cache) DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache cov, age = DataRetrieverService._retrieve_cache[dataset_ids[0]] # Verify that it was hit and it's now in there self.assertTrue(dataset_ids[0] in DataRetrieverService._retrieve_cache) gevent.sleep(DataRetrieverService._refresh_interval + 0.2) DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache cov, age2 = DataRetrieverService._retrieve_cache[dataset_ids[0]] self.assertTrue(age2 != age) for dataset_id in dataset_ids: DataRetrieverService._get_coverage(dataset_id) self.assertTrue( dataset_ids[0] not in DataRetrieverService._retrieve_cache) stream_id, route, stream_def, dataset_id = datasets[0] self.start_ingestion(stream_id, dataset_id) DataRetrieverService._get_coverage(dataset_id) self.assertTrue(dataset_id in DataRetrieverService._retrieve_cache) DataRetrieverService._refresh_interval = 100 self.publish_hifi(stream_id, route, 1) self.wait_until_we_have_enough_granules(dataset_id, data_size=20) event = gevent.event.Event() with gevent.Timeout(20): while not event.wait(0.1): if dataset_id not in DataRetrieverService._retrieve_cache: event.set() self.assertTrue(event.is_set()) @unittest.skip('Outdated due to ingestion retry') @attr('LOCOINT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Host requires file-system access to coverage files, CEI mode does not support.' ) def test_ingestion_failover(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) self.start_ingestion(stream_id, dataset_id) event = Event() def cb(*args, **kwargs): event.set() sub = EventSubscriber(event_type="ExceptionEvent", callback=cb, origin="stream_exception") sub.start() self.publish_fake_data(stream_id, route) self.wait_until_we_have_enough_granules(dataset_id, 40) file_path = DatasetManagementService._get_coverage_path(dataset_id) master_file = os.path.join(file_path, '%s_master.hdf5' % dataset_id) with open(master_file, 'w') as f: f.write('this will crash HDF') self.publish_hifi(stream_id, route, 5) self.assertTrue(event.wait(10)) sub.stop()
class SubProcess(Greenlet): """ Threaded execution of a command being called. """ def __init__(self, command, timeout=None): """ Initialize the function """ Greenlet.__init__(self, run=None) # we abort if this is set self._abort = Event() # this is set when an command has completed execution self._done = Event() # Tracks the PID file of item being executed self._pid = None # The return code is set after the programs execution self._returncode = ReturnCode.Unknown # The command itself should a list() identifing the executable as the # first entry followed by all of the arguments you wish to pass into # it. self._cmd = command # Since we need to poll until the execution of the process is # complete, we need to set a poll time. self._throttle = 0.5 # Track when the execution started self._execution_begin = None # Track when the execution completed self._execution_finish = None # The number of seconds at most we will allow the execution of the # process to run for before we force it to abort it's operation. # Setting this to zero disables this timeout restriction self._timeout = 0.0 if timeout: self._timeout = timeout # These are populated with the output of the stdout and # stderr stream. self._stdout = StringIO() self._stderr = StringIO() def elapsed(self): """ Returns the elapsed time (as a float) of the threaded execution which includes the number of microseconds. """ if self._execution_begin is None: # No elapsed time has taken place yet return 0.0 if self._execution_finish is not None: # Execution has completed, we only want to calculate # the execution time. elapsed_time = self._execution_finish - self._execution_begin else: # Calculate Elapsed Time elapsed_time = datetime.utcnow() - self._execution_begin elapsed_time = (elapsed_time.days * 86400) \ + elapsed_time.seconds \ + (elapsed_time.microseconds/1e6) return elapsed_time def _run(self): """ Read from the work_queue, process it using an NNTPRequest object. """ # Make sure our done flag is not set self._done.clear() # Execute our Process p1 = subprocess.Popen( self._cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) # Calculate Current Time self._execution_begin = datetime.utcnow() # Store some information self._pid = p1.pid # Calculate Wait Time max_wait_time = self._execution_begin + \ timedelta(seconds=self._timeout) while p1.poll() is None and not self._abort.is_set(): # Head of Poll Loop if self._timeout and \ datetime.utcnow() >= max_wait_time: # Process aborted (took too long) try: kill(self._pid, signal.SIGKILL) except: pass # Force bad return code self._returncode = ReturnCode.Timeout # Set our elapsed time to now self._execution_finish = datetime.utcnow() # Retrieve stdout/stderr self._stdout = StringIO(p1.stdout.read()) self._stderr = StringIO(p1.stderr.read()) # Make sure no one uses the PID anymore self._pid = None # Set our done flag self._done.set() return # CPU Throttle self._abort.wait(self._throttle) if p1.poll() is None or self._abort.is_set(): # Safety try: kill(self._pid, signal.SIGKILL) except: pass # Force bad return code self._returncode = ReturnCode.Aborted else: # Store return code self._returncode = p1.returncode # Execution Completion Time self._execution_finish = datetime.utcnow() # Retrieve stdout/stderr self._stdout = StringIO(p1.stdout.read()) self._stderr = StringIO(p1.stderr.read()) # Make sure no one uses the PID anymore self._pid = None # Set our done flag self._done.set() # We're done! return def is_complete(self, timeout=None): """ Returns True if the process has completed its execution if timeout is set to a time, then the function blocks up until that period of time elapses or the call completes. Times should be specified as float values (in seconds). """ if timeout is not None: self._done.wait(timeout) return self._done.is_set() def response_code(self): """ Returns the result """ return self._returncode def successful(self): """ Returns True if the calling action was successful or not. This call can be subjective because it bases it's response simply on whether or not a zero (0) was returned by the program called. Usually a non-zero value means there was a failure. """ return self._returncode is 0 def stdout(self, as_list=True): """ if as_list is set to True, then the stdout results are split on new lines into a list object """ # Ensure we're at the head of our buffer self._stdout.seek(0L, SEEK_SET) if as_list: return NEW_LINE_RE.split(self._stdout.read()) return self._stdout.read() def stderr(self, as_list=True): """ if as_list is set to True, then the stdout results are split on new lines into a list object """ # Ensure we're at the head of our buffer self._stderr.seek(0L, SEEK_SET) if as_list: return NEW_LINE_RE.split(self._stderr.read()) return self._stderr.read() def pid(self): """ returns the pid number of the running process, but returns None if the process is no longer running. """ return self._pid def abort(self): """ Abort the executing command """ self._abort.set() try: kill(self._pid, signal.SIGKILL) except: pass if self._pid: self.join(timeout=10.0) def __str__(self): """ returns the command being executed """ return ' '.join(self._cmd) def __repr__(self): """ Return a printable version of the file being read """ return '<SubProcess cmd=%s execution_time=%ds return_code=%d />' % ( self._cmd[0], self.elapsed(), self._returncode, )
class Channel(object): __slots__ = ('_recvq', '_closed', '_mon') def __init__(self): self._mon = Publisher() self._recvq = Queue() self._closed = Event() def __iter__(self): while not self.closed: msg = self.recv() if msg is StopIteration: break yield msg def __len__(self): """ Number of messages in buffer which haven't been delivered to watchers """ return self._recvq.qsize() def __del__(self): self.close() def send(self, msg): self._recvq.put_nowait(msg) if len(self._mon): self.recv() def watch(self): was_first = len(self._mon) == 0 subscriber = self._mon.subscribe() if was_first: self._recvall() return subscriber def _recvall(self): while self._recvq.qsize(): self.recv() def write(self, data): self.send(dict(data=data)) def recv(self): if self.closed: # XXX: raise better exception raise RuntimeError("Closed") msg = self._recvq.get() self._mon.send(msg) if msg is StopIteration: self._closed.set() return msg def wait(self): return self._closed.wait() @property def closed(self): return self._closed.ready() def close(self): if not self.closed: self.send(StopIteration) def datastream(self): return DataStream(self)
class BaseServer(object): """ An abstract base class that implements some common functionality for the servers in gevent. :param listener: Either be an address that the server should bind on or a :class:`gevent.socket.socket` instance that is already bound (and put into listening mode in case of TCP socket). :keyword handle: If given, the request handler. The request handler can be defined in a few ways. Most commonly, subclasses will implement a ``handle`` method as an instance method. Alternatively, a function can be passed as the ``handle`` argument to the constructor. In either case, the handler can later be changed by calling :meth:`set_handle`. When the request handler returns, the socket used for the request will be closed. :keyword spawn: If provided, is called to create a new greenlet to run the handler. By default, :func:`gevent.spawn` is used (meaning there is no artificial limit on the number of concurrent requests). Possible values for *spawn*: - a :class:`gevent.pool.Pool` instance -- ``handle`` will be executed using :meth:`gevent.pool.Pool.spawn` only if the pool is not full. While it is full, no new connections are accepted; - :func:`gevent.spawn_raw` -- ``handle`` will be executed in a raw greenlet which has a little less overhead then :class:`gevent.Greenlet` instances spawned by default; - ``None`` -- ``handle`` will be executed right away, in the :class:`Hub` greenlet. ``handle`` cannot use any blocking functions as it would mean switching to the :class:`Hub`. - an integer -- a shortcut for ``gevent.pool.Pool(integer)`` .. versionchanged:: 1.1a1 When the *handle* function returns from processing a connection, the client socket will be closed. This resolves the non-deterministic closing of the socket, fixing ResourceWarnings under Python 3 and PyPy. """ # pylint: disable=too-many-instance-attributes,bare-except,broad-except #: the number of seconds to sleep in case there was an error in accept() call #: for consecutive errors the delay will double until it reaches max_delay #: when accept() finally succeeds the delay will be reset to min_delay again min_delay = 0.01 max_delay = 1 #: Sets the maximum number of consecutive accepts that a process may perform on #: a single wake up. High values give higher priority to high connection rates, #: while lower values give higher priority to already established connections. #: Default is 100. Note, that in case of multiple working processes on the same #: listening value, it should be set to a lower value. (pywsgi.WSGIServer sets it #: to 1 when environ["wsgi.multiprocess"] is true) max_accept = 100 _spawn = Greenlet.spawn #: the default timeout that we wait for the client connections to close in stop() stop_timeout = 1 fatal_errors = (errno.EBADF, errno.EINVAL, errno.ENOTSOCK) def __init__(self, listener, handle=None, spawn='default'): self._stop_event = Event() self._stop_event.set() self._watcher = None self._timer = None self._handle = None # XXX: FIXME: Subclasses rely on the presence or absence of the # `socket` attribute to determine whether we are open/should be opened. # Instead, have it be None. self.pool = None try: self.set_listener(listener) self.set_spawn(spawn) self.set_handle(handle) self.delay = self.min_delay self.loop = get_hub().loop if self.max_accept < 1: raise ValueError('max_accept must be positive int: %r' % (self.max_accept, )) except: self.close() raise def set_listener(self, listener): if hasattr(listener, 'accept'): if hasattr(listener, 'do_handshake'): raise TypeError('Expected a regular socket, not SSLSocket: %r' % (listener, )) self.family = listener.family self.address = listener.getsockname() self.socket = listener else: self.family, self.address = parse_address(listener) def set_spawn(self, spawn): if spawn == 'default': self.pool = None self._spawn = self._spawn elif hasattr(spawn, 'spawn'): self.pool = spawn self._spawn = spawn.spawn elif isinstance(spawn, integer_types): from gevent.pool import Pool self.pool = Pool(spawn) self._spawn = self.pool.spawn else: self.pool = None self._spawn = spawn if hasattr(self.pool, 'full'): self.full = self.pool.full if self.pool is not None: self.pool._semaphore.rawlink(self._start_accepting_if_started) def set_handle(self, handle): if handle is not None: self.handle = handle if hasattr(self, 'handle'): self._handle = self.handle else: raise TypeError("'handle' must be provided") def _start_accepting_if_started(self, _event=None): if self.started: self.start_accepting() def start_accepting(self): if self._watcher is None: # just stop watcher without creating a new one? self._watcher = self.loop.io(self.socket.fileno(), 1) self._watcher.start(self._do_read) def stop_accepting(self): if self._watcher is not None: self._watcher.stop() self._watcher = None if self._timer is not None: self._timer.stop() self._timer = None def do_handle(self, *args): spawn = self._spawn handle = self._handle close = self.do_close try: if spawn is None: _handle_and_close_when_done(handle, close, args) else: spawn(_handle_and_close_when_done, handle, close, args) except: close(*args) raise def do_close(self, *args): pass def _do_read(self): for _ in xrange(self.max_accept): if self.full(): self.stop_accepting() return try: args = self.do_read() self.delay = self.min_delay if not args: return except: self.loop.handle_error(self, *sys.exc_info()) ex = sys.exc_info()[1] if self.is_fatal_error(ex): self.close() sys.stderr.write('ERROR: %s failed with %s\n' % (self, str(ex) or repr(ex))) return if self.delay >= 0: self.stop_accepting() self._timer = self.loop.timer(self.delay) self._timer.start(self._start_accepting_if_started) self.delay = min(self.max_delay, self.delay * 2) break else: try: self.do_handle(*args) except: self.loop.handle_error((args[1:], self), *sys.exc_info()) if self.delay >= 0: self.stop_accepting() self._timer = self.loop.timer(self.delay) self._timer.start(self._start_accepting_if_started) self.delay = min(self.max_delay, self.delay * 2) break def full(self): # copied from self.pool # pylint: disable=method-hidden return False def __repr__(self): return '<%s at %s %s>' % (type(self).__name__, hex(id(self)), self._formatinfo()) def __str__(self): return '<%s %s>' % (type(self).__name__, self._formatinfo()) def _formatinfo(self): if hasattr(self, 'socket'): try: fileno = self.socket.fileno() except Exception as ex: fileno = str(ex) result = 'fileno=%s ' % fileno else: result = '' try: if isinstance(self.address, tuple) and len(self.address) == 2: result += 'address=%s:%s' % self.address else: result += 'address=%s' % (self.address, ) except Exception as ex: result += str(ex) or '<error>' handle = self.__dict__.get('handle') if handle is not None: fself = getattr(handle, '__self__', None) try: if fself is self: # Checks the __self__ of the handle in case it is a bound # method of self to prevent recursivly defined reprs. handle_repr = '<bound method %s.%s of self>' % ( self.__class__.__name__, handle.__name__, ) else: handle_repr = repr(handle) result += ' handle=' + handle_repr except Exception as ex: result += str(ex) or '<error>' return result @property def server_host(self): """IP address that the server is bound to (string).""" if isinstance(self.address, tuple): return self.address[0] @property def server_port(self): """Port that the server is bound to (an integer).""" if isinstance(self.address, tuple): return self.address[1] def init_socket(self): """If the user initialized the server with an address rather than socket, then this function will create a socket, bind it and put it into listening mode. It is not supposed to be called by the user, it is called by :meth:`start` before starting the accept loop.""" pass @property def started(self): return not self._stop_event.is_set() def start(self): """Start accepting the connections. If an address was provided in the constructor, then also create a socket, bind it and put it into the listening mode. """ self.init_socket() self._stop_event.clear() try: self.start_accepting() except: self.close() raise def close(self): """Close the listener socket and stop accepting.""" self._stop_event.set() try: self.stop_accepting() finally: try: self.socket.close() except Exception: pass finally: self.__dict__.pop('socket', None) self.__dict__.pop('handle', None) self.__dict__.pop('_handle', None) self.__dict__.pop('_spawn', None) self.__dict__.pop('full', None) if self.pool is not None: self.pool._semaphore.unlink(self._start_accepting_if_started) @property def closed(self): return not hasattr(self, 'socket') def stop(self, timeout=None): """ Stop accepting the connections and close the listening socket. If the server uses a pool to spawn the requests, then :meth:`stop` also waits for all the handlers to exit. If there are still handlers executing after *timeout* has expired (default 1 second, :attr:`stop_timeout`), then the currently running handlers in the pool are killed. If the server does not use a pool, then this merely stops accepting connections; any spawned greenlets that are handling requests continue running until they naturally complete. """ self.close() if timeout is None: timeout = self.stop_timeout if self.pool: self.pool.join(timeout=timeout) self.pool.kill(block=True, timeout=1) def serve_forever(self, stop_timeout=None): """Start the server if it hasn't been already started and wait until it's stopped.""" # add test that serve_forever exists on stop() if not self.started: self.start() try: self._stop_event.wait() finally: Greenlet.spawn(self.stop, timeout=stop_timeout).join() def is_fatal_error(self, ex): return isinstance(ex, _socket.error) and ex.args[0] in self.fatal_errors
class GeventSelector(_BaseSelectorImpl): """ A selector implementation using gevent primitives. This is a type of :class:`selectors.BaseSelector`, so the documentation for that class applies here. .. caution:: As the base class indicates, it is critically important to unregister file objects before closing them. (Or close the selector they are registered with before closing them.) Failure to do so may crash the process or have other unintended results. """ # Notes on the approach: # # It's easy to wrap a selector implementation around # ``gevent.select.poll``; in fact that's what happens by default # when monkey-patching in Python 3. But the problem with that is # each call to ``selector.select()`` will result in creating and # then destroying new kernel-level polling resources, as nothing # in ``gevent.select`` can keep watchers around (because the underlying # file could be closed at any time). This ends up producing a large # number of syscalls that are unnecessary. # # So here, we take advantage of the fact that it is documented and # required that files not be closed while they are registered. # This lets us persist watchers. Indeed, it lets us continually # accrue events in the background before a call to ``select()`` is even # made. We can take advantage of this to return results immediately, without # a syscall, if we have them. # # We create watchers in ``register()`` and destroy them in # ``unregister()``. They do not get started until the first call # to ``select()``, though. Once they are started, they don't get # stopped until they deliver an event. # Lifecycle: # register() -> inactive_watchers # select() -> inactive_watchers -> active_watchers; # active_watchers -> inactive_watchers def __init__(self, hub=None): if hub is not None: self.hub = hub # {fd: watcher} self._active_watchers = {} self._inactive_watchers = {} # {fd: EVENT_READ|EVENT_WRITE} self._accumulated_events = defaultdict(int) self._ready = Event() super(GeventSelector, self).__init__() def __callback(self, events, fd): if events > 0: cur_event_for_fd = self._accumulated_events[fd] if events & _EV_READ: cur_event_for_fd |= EVENT_READ if events & _EV_WRITE: cur_event_for_fd |= EVENT_WRITE self._accumulated_events[fd] = cur_event_for_fd self._ready.set() @Lazy def hub(self): # pylint:disable=method-hidden return get_hub() def register(self, fileobj, events, data=None): key = _BaseSelectorImpl.register(self, fileobj, events, data) if events == _ALL_EVENTS: flags = _POLL_ALL elif events == EVENT_READ: flags = _EV_READ else: flags = _EV_WRITE loop = self.hub.loop io = loop.io MAXPRI = loop.MAXPRI self._inactive_watchers[key.fd] = watcher = io(key.fd, flags) watcher.priority = MAXPRI return key def unregister(self, fileobj): key = _BaseSelectorImpl.unregister(self, fileobj) if key.fd in self._active_watchers: watcher = self._active_watchers.pop(key.fd) else: watcher = self._inactive_watchers.pop(key.fd) watcher.stop() watcher.close() self._accumulated_events.pop(key.fd, None) return key # XXX: Can we implement ``modify`` more efficiently than # ``unregister()``+``register()``? We could detect the no-change # case and do nothing; recent versions of the standard library # do that. def select(self, timeout=None): """ Poll for I/O. Note that, like the built-in selectors, this will block indefinitely if no timeout is given and no files have been registered. """ # timeout > 0 : block seconds # timeout <= 0 : No blocking. # timeout = None: Block forever # Event.wait doesn't deal with negative values if timeout is not None and timeout < 0: timeout = 0 # Start any watchers that need started. Note that they may # not actually get a chance to do anything yet if we already had # events set. for fd, watcher in iteritems(self._inactive_watchers): watcher.start(self.__callback, fd, pass_events=True) self._active_watchers.update(self._inactive_watchers) self._inactive_watchers.clear() # The _ready event is either already set (in which case # there are some results waiting in _accumulated_events) or # not set, in which case we have to block. But to make the two cases # behave the same, we will always yield to the event loop. if self._ready.is_set(): sleep() self._ready.wait(timeout) self._ready.clear() # TODO: If we have nothing ready, but they ask us not to block, # should we make an effort to actually spin the event loop and let # it check for events? result = [] for fd, event in iteritems(self._accumulated_events): key = self._key_from_fd(fd) watcher = self._active_watchers.pop(fd) ## The below is taken without comment from ## https://github.com/gevent/gevent/pull/1523/files and ## hasn't been checked: # # Since we are emulating an epoll object within another epoll object, # once a watcher has fired, we must deactivate it until poll is called # next. If we did not, someone else could call, e.g., gevent.time.sleep # and any unconsumed bytes on our watched fd would prevent the process # from sleeping correctly. watcher.stop() if key: result.append((key, event & key.events)) self._inactive_watchers[fd] = watcher else: # pragma: no cover # If the key was gone, then somehow we've been unregistered. # Don't put it back in inactive, close it. watcher.close() self._accumulated_events.clear() return result def close(self): for d in self._active_watchers, self._inactive_watchers: if d is None: continue # already closed for watcher in itervalues(d): watcher.stop() watcher.close() self._active_watchers = self._inactive_watchers = None self._accumulated_events = None self.hub = None _BaseSelectorImpl.close(self)
class TestTransforms(IonIntegrationTestCase): def setUp(self): self._start_container() self.queue_cleanup = [] self.exchange_cleanup = [] def tearDown(self): for queue in self.queue_cleanup: xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() for exchange in self.exchange_cleanup: xp = self.container.ex_manager.create_xp(exchange) xp.delete() def test_stats(self): self.container.spawn_process('test', 'ion.core.process.transform', 'TransformBase', {}, 'test_transform') test_transform = self.container.proc_manager.procs['test_transform'] test_transform._stats['hits'] = 100 retval = TransformBase.stats('test_transform') self.assertEquals(retval, {'hits': 100}) def test_stream_transforms(self): self.verified = Event() input_route = StreamRoute('test_exchange', 'input') output_route = StreamRoute('test_exchange', 'output') def verify(m, route, stream_id): self.assertEquals(route, output_route) self.assertEquals(m, 'test') self.verified.set() # Create I/O Processes #-------------------------------------------------------------------------------- pub_proc = TransformBase() pub_proc.container = self.container publisher = StreamPublisher(process=pub_proc, stream_route=input_route) transform = self.container.spawn_process( 'transform', 'ion.core.process.test.test_transform', 'EmptyDataProcess', { 'process': { 'queue_name': 'transform_input', 'exchange_point': output_route.exchange_point, 'routing_key': output_route.routing_key } }, 'transformpid') transform = self.container.proc_manager.procs[transform] sub_proc = TransformBase() sub_proc.container = self.container subscriber = StreamSubscriber(process=sub_proc, exchange_name='subscriber', callback=verify) # Bind the transports #-------------------------------------------------------------------------------- transform.subscriber.xn.bind(input_route.routing_key, publisher.xp) subscriber.xn.bind(output_route.routing_key, transform.publisher.xp) subscriber.start() self.addCleanup(subscriber.stop) publisher.publish('test') self.assertTrue(self.verified.wait(4))
def main(default_config_file, is_gateway): global logger import futile.logging logger = futile.logging.get_logger(__name__) from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter config_locations = (".", "/etc/openmtc/gevent", "/etc/openmtc") parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-f", "--configfile", default=None, help="Location of the configuration file. If " "unspecified the system will look for a file called %s" " in these locations: %s" % (default_config_file, ', '.join(config_locations))) parser.add_argument("-v", "--verbose", action="count", default=None, help="Increase verbosity in output. This option can be" " specified multiple times.") parser.add_argument("--profiler", action="store_true", help="Use GreenletProfiler") args = parser.parse_args() configfile = args.configfile futile.logging.set_default_level(futile.logging.DEBUG) try: if not configfile: import os.path for d in config_locations: configfile = os.path.join(os.path.abspath(d), default_config_file) logger.debug("Trying config file location: %s", configfile) if os.path.isfile(configfile): break else: raise ConfigurationError( "Configuration file %s not found in " "any of these locations: %s" % (default_config_file, config_locations)) config = load_config(configfile) except ConfigurationError as e: sys.stderr.write(str(e) + "\n") sys.exit(2) import openmtc_cse.api openmtc_cse.api.config = config import openmtc_server.api openmtc_server.api.config = config # TODO: kca: # Also: set global (non-futile) level? if "logging" in config: # TODO init logging # FIXME: This won't work, needs translation to log levels log_conf = config["logging"] if args.verbose is None: futile.logging.set_default_level( log_conf.get("level") or futile.logging.WARNING) elif args.verbose >= 2: futile.logging.set_default_level(futile.logging.DEBUG) else: futile.logging.set_default_level(futile.logging.INFO) logfile = log_conf.get("file") if logfile: futile.logging.add_log_file(logfile) else: futile.logging.set_default_level(futile.logging.DEBUG) # make iso8601 logging shut up logger = futile.logging.get_logger(__name__) futile.logging.get_logger("iso8601").setLevel(futile.logging.ERROR) logger.debug("Running OpenMTC") from itertools import starmap import signal from gevent import spawn_later from gevent.event import Event as GEventEvent from openmtc_gevent.TaskRunner import GEventTaskRunner from openmtc_cse.methoddomain import OneM2MMethodDomain from openmtc_cse.transport import OneM2MTransportDomain from openmtc_server.platform.default.Event import (ResourceFinishEvent, NetworkEvent) from GEventNetworkManager import GEventNetworkManager from openmtc_server.util.db import load_db_module omd = OneM2MMethodDomain(config=config) otd = OneM2MTransportDomain(config=config) nm = GEventNetworkManager(config=config.get("network_manager", {})) task_runner = GEventTaskRunner() _components.append(task_runner) _timers = set() db = load_db_module(config) class Api(object): PLATFORM = "gevent" class events(object): resource_created = ResourceFinishEvent(task_runner.run_task) resource_deleted = ResourceFinishEvent(task_runner.run_task) resource_updated = ResourceFinishEvent(task_runner.run_task) resource_announced = ResourceFinishEvent(task_runner.run_task) # fired when a network interface appeared # called with <interface> interface_created = NetworkEvent(task_runner.run_task) # fired when a network interface was disappeared # called with <interface> interface_removed = NetworkEvent(task_runner.run_task) # fired when an address appeared on an existing interface # called with <interface>, <address> address_created = NetworkEvent(task_runner.run_task) # fired when an address disappeared on an existing interface # called with <interface>, <address> address_removed = NetworkEvent(task_runner.run_task) start_onem2m_session = db.start_onem2m_session get_shelve = db.get_shelve # handle request handle_onem2m_request = omd.handle_onem2m_request # send request send_onem2m_request = otd.send_onem2m_request send_notify = otd.send_notify register_point_of_access = otd.register_point_of_access # connectors and endpoints register_onem2m_client = otd.register_client get_onem2m_endpoints = otd.get_endpoints add_poa_list = otd.add_poa_list network_manager = nm run_task = task_runner.run_task @staticmethod def set_timer(t, f, *args, **kw): timer = None def wrapper(): _timers.discard(timer) f(*args, **kw) timer = spawn_later(t, wrapper) _timers.add(timer) return timer @staticmethod def cancel_timer(timer): _timers.discard(timer) timer.kill() map = map @staticmethod def starmap(c, l): return tuple(starmap(c, l)) Api.db = db openmtc_cse.api.api = Api openmtc_cse.api.events = Api.events openmtc_server.api.api = Api openmtc_server.api.events = Api.events shutdown_event = GEventEvent() gevent.signal(signal.SIGTERM, shutdown_event.set) gevent.signal(signal.SIGINT, shutdown_event.set) try: init_component(otd, Api) init_component(omd, Api) init_component(nm, Api) force = config["database"].get("dropDB") if force or not db.is_initialized(): db.initialize(force) omd.init_cse_base() omd.start() load_plugins(Api, config.get("plugins", ()), config["global"], config["onem2m"], is_gateway) init_plugins() start_plugins() logger.info("OpenMTC is running") except: logger.exception("Error during startup") else: if args.profiler: import GreenletProfiler GreenletProfiler.set_clock_type("cpu") GreenletProfiler.start() # wait for shutdown event shutdown_event.wait() if args.profiler: GreenletProfiler.stop() stats = GreenletProfiler.get_func_stats() stats.print_all() stats.save('profile.callgrind', type='callgrind') stop_plugins() stop_components() for timer in _timers: try: timer.kill() except: logger.exception("Failed to kill timer %s", timer)
class Auction(DBServiceMixin, RequestIDServiceMixin, AuditServiceMixin, BiddersServiceMixin, DateTimeServiceMixin, StagesServiceMixin, PostAuctionServiceMixin): """Auction Worker Class""" def __init__(self, tender_id, worker_defaults, auction_data={}, lot_id=None): super(Auction, self).__init__() self.generate_request_id() self.tender_id = tender_id self.lot_id = lot_id if lot_id: self.auction_doc_id = tender_id + "_" + lot_id else: self.auction_doc_id = tender_id self.tender_url = urljoin( worker_defaults["resource_api_server"], '/api/{0}/{1}/{2}'.format(worker_defaults["resource_api_version"], worker_defaults["resource_name"], tender_id)) if auction_data: self.debug = True LOGGER.setLevel(logging.DEBUG) self._auction_data = auction_data else: self.debug = False self._end_auction_event = Event() self.bids_actions = BoundedSemaphore() self.session = RequestsSession() self.worker_defaults = worker_defaults if self.worker_defaults.get('with_document_service', False): self.session_ds = RequestsSession() self._bids_data = {} self.db = Database(str(self.worker_defaults["COUCH_DATABASE"]), session=Session(retry_delays=range(10))) self.audit = {} self.retries = 10 self.bidders_count = 0 self.bidders_data = [] self.bidders_features = {} self.bidders_coeficient = {} self.features = None self.mapping = {} self.rounds_stages = [] def schedule_auction(self): self.generate_request_id() self.get_auction_document() if self.debug: LOGGER.info("Get _auction_data from auction_document") self._auction_data = self.auction_document.get( 'test_auction_data', {}) self.get_auction_info() self.prepare_audit() self.prepare_auction_stages() self.save_auction_document() round_number = 0 SCHEDULER.add_job(self.start_auction, 'date', kwargs={"switch_to_round": round_number}, run_date=self.convert_datetime( self.auction_document['stages'][0]['start']), name="Start of Auction", id="Start of Auction") round_number += 1 SCHEDULER.add_job(self.end_first_pause, 'date', kwargs={"switch_to_round": round_number}, run_date=self.convert_datetime( self.auction_document['stages'][1]['start']), name="End of Pause Stage: [0 -> 1]", id="End of Pause Stage: [0 -> 1]") round_number += 1 for index in xrange(2, len(self.auction_document['stages'])): if self.auction_document['stages'][index - 1]['type'] == 'bids': SCHEDULER.add_job( self.end_bids_stage, 'date', kwargs={"switch_to_round": round_number}, run_date=self.convert_datetime( self.auction_document['stages'][index]['start']), name="End of Bids Stage: [{} -> {}]".format( index - 1, index), id="End of Bids Stage: [{} -> {}]".format( index - 1, index)) elif self.auction_document['stages'][index - 1]['type'] == 'pause': SCHEDULER.add_job( self.next_stage, 'date', kwargs={"switch_to_round": round_number}, run_date=self.convert_datetime( self.auction_document['stages'][index]['start']), name="End of Pause Stage: [{} -> {}]".format( index - 1, index), id="End of Pause Stage: [{} -> {}]".format( index - 1, index)) round_number += 1 LOGGER.info("Prepare server ...", extra={ "JOURNAL_REQUEST_ID": self.request_id, "MESSAGE_ID": AUCTION_WORKER_SERVICE_PREPARE_SERVER }) self.server = run_server( self, self.convert_datetime( self.auction_document['stages'][-2]['start']), LOGGER) def wait_to_end(self): self._end_auction_event.wait() LOGGER.info("Stop auction worker", extra={ "JOURNAL_REQUEST_ID": self.request_id, "MESSAGE_ID": AUCTION_WORKER_SERVICE_STOP_AUCTION_WORKER }) def start_auction(self, switch_to_round=None): self.generate_request_id() self.audit['timeline']['auction_start']['time'] = datetime.now( tzlocal()).isoformat() LOGGER.info('---------------- Start auction ----------------', extra={ "JOURNAL_REQUEST_ID": self.request_id, "MESSAGE_ID": AUCTION_WORKER_SERVICE_START_AUCTION }) self.get_auction_info() self.get_auction_document() # Initital Bids bids = deepcopy(self.bidders_data) self.auction_document["initial_bids"] = [] bids_info = sorting_start_bids_by_amount(bids, features=self.features) for index, bid in enumerate(bids_info): amount = bid["value"]["amount"] audit_info = { "bidder": bid["id"], "date": bid["date"], "amount": amount } if self.features: amount_features = cooking(amount, self.features, self.bidders_features[bid["id"]]) coeficient = self.bidders_coeficient[bid["id"]] audit_info["amount_features"] = str(amount_features) audit_info["coeficient"] = str(coeficient) else: coeficient = None amount_features = None self.audit['timeline']['auction_start']['initial_bids'].append( audit_info) self.auction_document["initial_bids"].append( prepare_initial_bid_stage( time=bid["date"] if "date" in bid else self.startDate, bidder_id=bid["id"], bidder_name=self.mapping[bid["id"]], amount=amount, coeficient=coeficient, amount_features=amount_features)) if isinstance(switch_to_round, int): self.auction_document["current_stage"] = switch_to_round else: self.auction_document["current_stage"] = 0 all_bids = deepcopy(self.auction_document["initial_bids"]) minimal_bids = [] for bid_info in self.bidders_data: minimal_bids.append( get_latest_bid_for_bidder(all_bids, str(bid_info['id']))) minimal_bids = self.filter_bids_keys(sorting_by_amount(minimal_bids)) self.update_future_bidding_orders(minimal_bids) self.save_auction_document() def end_first_pause(self, switch_to_round=None): self.generate_request_id() LOGGER.info('---------------- End First Pause ----------------', extra={ "JOURNAL_REQUEST_ID": self.request_id, "MESSAGE_ID": AUCTION_WORKER_SERVICE_END_FIRST_PAUSE }) self.bids_actions.acquire() self.get_auction_document() if isinstance(switch_to_round, int): self.auction_document["current_stage"] = switch_to_round else: self.auction_document["current_stage"] += 1 self.save_auction_document() self.bids_actions.release() def end_auction(self): LOGGER.info('---------------- End auction ----------------', extra={ "JOURNAL_REQUEST_ID": self.request_id, "MESSAGE_ID": AUCTION_WORKER_SERVICE_END_AUCTION }) LOGGER.debug("Stop server", extra={"JOURNAL_REQUEST_ID": self.request_id}) if self.server: self.server.stop() LOGGER.debug("Clear mapping", extra={"JOURNAL_REQUEST_ID": self.request_id}) delete_mapping(self.worker_defaults, self.auction_doc_id) start_stage, end_stage = self.get_round_stages(ROUNDS) minimal_bids = deepcopy( self.auction_document["stages"][start_stage:end_stage]) minimal_bids = self.filter_bids_keys(sorting_by_amount(minimal_bids)) self.auction_document["results"] = [] for item in minimal_bids: self.auction_document["results"].append( prepare_results_stage(**item)) self.auction_document["current_stage"] = ( len(self.auction_document["stages"]) - 1) LOGGER.debug(' '.join(('Document in end_stage: \n', yaml_dump(dict(self.auction_document)))), extra={"JOURNAL_REQUEST_ID": self.request_id}) self.approve_audit_info_on_announcement() LOGGER.info('Audit data: \n {}'.format(yaml_dump(self.audit)), extra={"JOURNAL_REQUEST_ID": self.request_id}) if self.debug: LOGGER.debug('Debug: put_auction_data disabled !!!', extra={"JOURNAL_REQUEST_ID": self.request_id}) sleep(10) self.save_auction_document() else: if self.put_auction_data(): self.save_auction_document() LOGGER.debug("Fire 'stop auction worker' event", extra={"JOURNAL_REQUEST_ID": self.request_id}) def cancel_auction(self): self.generate_request_id() if self.get_auction_document(): LOGGER.info( "Auction {} canceled".format(self.auction_doc_id), extra={'MESSAGE_ID': AUCTION_WORKER_SERVICE_AUCTION_CANCELED}) self.auction_document["current_stage"] = -100 self.auction_document["endDate"] = datetime.now( tzlocal()).isoformat() LOGGER.info("Change auction {} status to 'canceled'".format( self.auction_doc_id), extra={ 'MESSAGE_ID': AUCTION_WORKER_SERVICE_AUCTION_STATUS_CANCELED }) self.save_auction_document() else: LOGGER.info( "Auction {} not found".format(self.auction_doc_id), extra={'MESSAGE_ID': AUCTION_WORKER_SERVICE_AUCTION_NOT_FOUND}) def reschedule_auction(self): self.generate_request_id() if self.get_auction_document(): LOGGER.info( "Auction {} has not started and will be rescheduled".format( self.auction_doc_id), extra={ 'MESSAGE_ID': AUCTION_WORKER_SERVICE_AUCTION_RESCHEDULE }) self.auction_document["current_stage"] = -101 self.save_auction_document() else: LOGGER.info( "Auction {} not found".format(self.auction_doc_id), extra={'MESSAGE_ID': AUCTION_WORKER_SERVICE_AUCTION_NOT_FOUND})
class RBDMirrorThrasher(Greenlet, Thrasher): """ RBDMirrorThrasher:: The RBDMirrorThrasher thrashes rbd-mirror daemons during execution of other tasks (workunits, etc). The config is optional. Many of the config parameters are a maximum value to use when selecting a random value from a range. The config is a dict containing some or all of: cluster: [default: ceph] cluster to thrash max_thrash: [default: 1] the maximum number of active rbd-mirror daemons per cluster will be thrashed at any given time. min_thrash_delay: [default: 60] minimum number of seconds to delay before thrashing again. max_thrash_delay: [default: 120] maximum number of seconds to delay before thrashing again. max_revive_delay: [default: 10] maximum number of seconds to delay before bringing back a thrashed rbd-mirror daemon. randomize: [default: true] enables randomization and use the max/min values seed: [no default] seed the random number generator Examples:: The following example disables randomization, and uses the max delay values: tasks: - ceph: - rbd_mirror_thrash: randomize: False max_thrash_delay: 10 """ def __init__(self, ctx, config, cluster, daemons): Greenlet.__init__(self) Thrasher.__init__(self, "RBDMirrorThrasher") self.ctx = ctx self.config = config self.cluster = cluster self.daemons = daemons self.logger = log self.name = 'thrasher.rbd_mirror.[{cluster}]'.format(cluster=cluster) self.stopping = Event() self.randomize = bool(self.config.get('randomize', True)) self.max_thrash = int(self.config.get('max_thrash', 1)) self.min_thrash_delay = float(self.config.get('min_thrash_delay', 60.0)) self.max_thrash_delay = float( self.config.get('max_thrash_delay', 120.0)) self.max_revive_delay = float(self.config.get('max_revive_delay', 10.0)) def _run(self): try: self.do_thrash() except Exception as e: # See _run exception comment for MDSThrasher self.exception = e self.logger.exception("exception:") # Allow successful completion so gevent doesn't see an exception. # The DaemonWatchdog will observe the error and tear down the test. def log(self, x): """Write data to logger assigned to this RBDMirrorThrasher""" self.logger.info(x) def stop(self): self.stopping.set() def do_thrash(self): """ Perform the random thrashing action """ self.log('starting thrash for cluster {cluster}'.format( cluster=self.cluster)) stats = { "kill": 0, } while not self.stopping.is_set(): delay = self.max_thrash_delay if self.randomize: delay = random.randrange(self.min_thrash_delay, self.max_thrash_delay) if delay > 0.0: self.log('waiting for {delay} secs before thrashing'.format( delay=delay)) self.stopping.wait(delay) if self.stopping.is_set(): continue killed_daemons = [] weight = 1.0 / len(self.daemons) count = 0 for daemon in self.daemons: skip = random.uniform(0.0, 1.0) if weight <= skip: self.log( 'skipping daemon {label} with skip ({skip}) > weight ({weight})' .format(label=daemon.id_, skip=skip, weight=weight)) continue self.log('kill {label}'.format(label=daemon.id_)) try: daemon.signal(signal.SIGTERM) except socket.error: pass killed_daemons.append(daemon) stats['kill'] += 1 # if we've reached max_thrash, we're done count += 1 if count >= self.max_thrash: break if killed_daemons: # wait for a while before restarting delay = self.max_revive_delay if self.randomize: delay = random.randrange(0.0, self.max_revive_delay) self.log( 'waiting for {delay} secs before reviving daemons'.format( delay=delay)) sleep(delay) for daemon in killed_daemons: self.log('waiting for {label}'.format(label=daemon.id_)) try: run.wait([daemon.proc], timeout=600) except CommandFailedError: pass except: self.log( 'Failed to stop {label}'.format(label=daemon.id_)) try: # try to capture a core dump daemon.signal(signal.SIGABRT) except socket.error: pass raise finally: daemon.reset() for daemon in killed_daemons: self.log('reviving {label}'.format(label=daemon.id_)) daemon.start() for stat in stats: self.log("stat['{key}'] = {value}".format(key=stat, value=stats[stat]))
class TestPreloadThenLoadDataset(IonIntegrationTestCase): """ Uses the preload system to define the ExternalDataset and related resources, then invokes services to perform the load """ def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') config = dict(op="load", scenario="NOSE", attachments="res/preload/r2_ioc/attachments") self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=config) self.pubsub = PubsubManagementServiceClient() self.dams = DataAcquisitionManagementServiceClient() def test_use_case(self): # setUp() has already started the container and performed the preload # self.assert_dataset_loaded('Test External CTD Dataset') # make sure we have the ExternalDataset resources self.assert_dataset_loaded( 'Unit Test SMB37') # association changed -- now use device name self.do_listen_for_incoming( ) # listen for any data being received from the dataset self.do_read_dataset() # call services to load dataset self.assert_data_received() # check that data was received as expected self.do_shutdown() def assert_dataset_loaded(self, name): rr = self.container.resource_registry # self.external_dataset = self.find_object_by_name(name, RT.ExternalDataset) devs, _ = rr.find_resources(RT.InstrumentDevice, name=name, id_only=False) self.assertEquals(len(devs), 1) self.device = devs[0] obj, _ = rr.find_objects(subject=self.device._id, predicate=PRED.hasAgentInstance, object_type=RT.ExternalDatasetAgentInstance) self.agent_instance = obj[0] obj, _ = rr.find_objects(object_type=RT.ExternalDatasetAgent, predicate=PRED.hasAgentDefinition, subject=self.agent_instance._id) self.agent = obj[0] driver_cfg = self.agent_instance.driver_config #stream_definition_id = driver_cfg['dh_cfg']['stream_def'] if 'dh_cfg' in driver_cfg else driver_cfg['stream_def'] #self.stream_definition = rr.read(stream_definition_id) self.data_product = rr.read_object(subject=self.device._id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct) self.dataset_id = rr.read_object(subject=self.data_product._id, predicate=PRED.hasDataset, object_type=RT.Dataset, id_only=True) ids, _ = rr.find_objects(subject=self.data_product._id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) self.stream_id = ids[0] self.route = self.pubsub.read_stream_route(self.stream_id) def do_listen_for_incoming(self): subscription_id = self.pubsub.create_subscription( 'validator', data_product_ids=[self.data_product._id]) self.addCleanup(self.pubsub.delete_subscription, subscription_id) self.granule_capture = [] self.granule_count = 0 def on_granule(msg, route, stream_id): self.granule_count += 1 if self.granule_count < 5: self.granule_capture.append(msg) validator = StandaloneStreamSubscriber('validator', callback=on_granule) validator.start() self.addCleanup(validator.stop) self.pubsub.activate_subscription(subscription_id) self.addCleanup(self.pubsub.deactivate_subscription, subscription_id) self.dataset_modified = Event() def cb2(*args, **kwargs): self.dataset_modified.set() # TODO: event isn't using the ExternalDataset, but a different ID for a Dataset es = EventSubscriber(event_type=OT.DatasetModified, callback=cb2, origin=self.dataset_id) es.start() self.addCleanup(es.stop) def do_read_dataset(self): self.dams.start_external_dataset_agent_instance( self.agent_instance._id) # # should i wait for process (above) to start # before launching client (below)? # self.client = None end = time.time() + MAX_AGENT_START_TIME while not self.client and time.time() < end: try: self.client = ResourceAgentClient(self.device._id, process=FakeProcess()) except NotFound: time.sleep(2) if not self.client: self.fail( msg='external dataset agent process did not start in %d seconds' % MAX_AGENT_START_TIME) self.client.execute_agent( AgentCommand(command=ResourceAgentEvent.INITIALIZE)) self.client.execute_agent( AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)) self.client.execute_agent(AgentCommand(command=ResourceAgentEvent.RUN)) self.client.execute_resource(command=AgentCommand( command=DriverEvent.START_AUTOSAMPLE)) def assert_data_received(self): #let it go for up to 120 seconds, then stop the agent and reset it if not self.dataset_modified.is_set(): self.dataset_modified.wait(30) self.assertTrue(self.granule_count > 2, msg='granule count = %d' % self.granule_count) rdt = RecordDictionaryTool.load_from_granule(self.granule_capture[0]) self.assertAlmostEqual(0, rdt['oxygen'][0], delta=0.01) self.assertAlmostEqual(309.77, rdt['pressure'][0], delta=0.01) self.assertAlmostEqual(37.9848, rdt['conductivity'][0], delta=0.01) self.assertAlmostEqual(9.5163, rdt['temp'][0], delta=0.01) self.assertAlmostEqual(3527207897.0, rdt['time'][0], delta=1) def do_shutdown(self): self.dams.stop_external_dataset_agent_instance(self.agent_instance._id)
class _FelixEtcdWatcher(gevent.Greenlet): """ Greenlet that communicates with the etcd driver over a socket. * Does the initial handshake with the driver, sending it the init message. * Receives the pre-loaded config from the driver and uses that to do Felix's one-off configuration. * Sends the relevant config back to the driver. * Processes the event stream from the driver, sending it on to the splitter. This class is similar to the EtcdWatcher class in that it uses a PathDispatcher to fan out updates but it doesn't own an etcd connection of its own. """ def __init__(self, config, etcd_api, status_reporter, hosts_ipset): super(_FelixEtcdWatcher, self).__init__() self._config = config self._etcd_api = etcd_api self._status_reporter = status_reporter self.hosts_ipset = hosts_ipset # Whether we've been in sync with etcd at some point. self._been_in_sync = False # Keep track of the config loaded from etcd so we can spot if it # changes. self.last_global_config = None self.last_host_config = None self.my_config_dir = dir_for_per_host_config(self._config.HOSTNAME) # Events triggered by the EtcdAPI Actor to tell us to load the config # and start polling. These are one-way flags. self.load_config = Event() self.begin_polling = Event() # Event that we trigger once the config is loaded. self.configured = Event() # Polling state initialized at poll start time. self.splitter = None # Next-hop IP addresses of our hosts, if populated in etcd. self.ipv4_by_hostname = {} # Forces a resync after the current poll if set. Safe to set from # another thread. Automatically reset to False after the resync is # triggered. self.resync_requested = False self.dispatcher = PathDispatcher() # The Popen object for the driver. self._driver_process = None # Stats. self.read_count = 0 self.msgs_processed = 0 self.last_rate_log_time = monotonic_time() # Register for events when values change. self._register_paths() self._usage_report_greenlet = gevent.Greenlet( self._periodically_usage_report) def estimated_host_count(self): """ Thread safe for caller because calling len() on a dict is atomic in Python. :return An aproximation to our Felix cluster count """ return len(self.ipv4_by_hostname) @logging_exceptions def _periodically_usage_report(self): """ Greenlet: periodically report the cluster existence to projectcalico.org. Period is about once per day. :return: Does not return, unless USAGE_REPORT disabled. """ interval = 86400 # Once every 24 hours minus 12 minute jitter jitter = random.random() * 0.01 * interval try: calico_version = str(pkg_resources.require("calico")[0].version) except ResolutionError: calico_version = "NA" _log.info( "Started usage report thread. Usage report interval: %s, pre-jitter: %s", interval, jitter) # Pre-Jitter the reporting thread start by 1% of interval (about 12 minutes) # Jitter prevents thundering herd for large clusters when the cluster first starts # Do pre-jitter only for clusters greater than 25. felix_count = self.estimated_host_count() if (felix_count >= 25): gevent.sleep(jitter) while True: config = self._config felix_count = self.estimated_host_count() cluster_type = "NA" if self._config.USAGE_REPORT: _log.info("usage report is enabled") report_usage_and_get_warnings(calico_version, config.HOSTNAME, config.CLUSTER_GUID, felix_count, cluster_type) # Jitter by 10% of interval (about 120 minutes) jitter = random.random() * 0.1 * interval sleep_time = interval - jitter _log.info("Usage report interval: %s, sleep-time: %s", interval, sleep_time) gevent.sleep(sleep_time) def _register_paths(self): """ Program the dispatcher with the paths we care about. """ reg = self.dispatcher.register # Profiles and their contents. reg(TAGS_KEY, on_set=self.on_tags_set, on_del=self.on_tags_delete) reg(RULES_KEY, on_set=self.on_rules_set, on_del=self.on_rules_delete) reg(PROFILE_LABELS_KEY, on_set=self.on_prof_labels_set, on_del=self.on_prof_labels_delete) # Tiered policy reg(TIER_DATA, on_set=self.on_tier_data_set, on_del=self.on_tier_data_delete) reg(TIERED_PROFILE, on_set=self.on_tiered_policy_set, on_del=self.on_tiered_policy_delete) # Hosts and endpoints. reg(HOST_IP_KEY, on_set=self.on_host_ip_set, on_del=self.on_host_ip_delete) reg(PER_ENDPOINT_KEY, on_set=self.on_endpoint_set, on_del=self.on_endpoint_delete) reg(HOST_IFACE_KEY, on_set=self.on_host_ep_set, on_del=self.on_host_ep_delete) reg(CIDR_V4_KEY, on_set=self.on_ipam_v4_pool_set, on_del=self.on_ipam_v4_pool_delete) # Configuration keys. If any of these is changed or created, we'll # restart to pick up the change. reg(CONFIG_PARAM_KEY, on_set=self._on_config_updated, on_del=self._on_config_updated) reg(PER_HOST_CONFIG_PARAM_KEY, on_set=self._on_host_config_updated, on_del=self._on_host_config_updated) @logging_exceptions def _run(self): # Don't do anything until we're told to load the config. _log.info("Waiting for load_config event...") self.load_config.wait() _log.info("...load_config set. Starting driver read %s loop", self) # Start the driver process and wait for it to connect back to our # socket. self._msg_reader, self._msg_writer = self._start_driver() # Loop reading from the socket and processing messages. self._loop_reading_from_driver() def _loop_reading_from_driver(self): while True: try: # Note: self._msg_reader.new_messages() returns iterator so # whole for loop must be inside the try. for msg_type, msg in self._msg_reader.new_messages(timeout=1): self._dispatch_msg_from_driver(msg_type, msg) except SocketClosed: _log.critical("The driver process closed its socket, Felix " "must exit.") die_and_restart() if self.resync_requested: _log.info("Resync requested, sending resync request to driver") self.resync_requested = False self._msg_writer.send_message(MSG_TYPE_RESYNC) # Check that the driver hasn't died. The recv() call should # raise an exception when the buffer runs dry but this usually # gets hit first. driver_rc = self._driver_process.poll() if driver_rc is not None: _log.critical( "Driver process died with RC = %s. Felix must " "exit.", driver_rc) die_and_restart() def _dispatch_msg_from_driver(self, msg_type, msg): # Optimization: put update first in the "switch" block because # it's on the critical path. if msg_type == MSG_TYPE_UPDATE: _stats.increment("Update messages from driver") self._on_update_from_driver(msg) elif msg_type == MSG_TYPE_CONFIG_LOADED: _stats.increment("Config loaded messages from driver") self._on_config_loaded_from_driver(msg) elif msg_type == MSG_TYPE_STATUS: _stats.increment("Status messages from driver") self._on_status_from_driver(msg) else: raise RuntimeError("Unexpected message %s" % msg) self.msgs_processed += 1 if self.msgs_processed % MAX_EVENTS_BEFORE_YIELD == 0: # Yield to ensure that other actors make progress. (gevent only # yields for us if the socket would block.) The sleep must be # non-zero to work around gevent issue where we could be # immediately rescheduled. gevent.sleep(0.000001) def _on_update_from_driver(self, msg): """ Called when the driver sends us a key/value pair update. After the initial handshake, the stream of events consists entirely of updates unless something happens to change the state of the driver. :param dict msg: The message received from the driver. """ assert self.configured.is_set(), "Received update before config" # The driver starts polling immediately, make sure we block until # everyone else is ready to receive updates. self.begin_polling.wait() # Unpack the message. key = msg[MSG_KEY_KEY] value = msg[MSG_KEY_VALUE] _log.debug("Update from driver: %s -> %s", key, value) # Output some very coarse stats. self.read_count += 1 if self.read_count % 1000 == 0: now = monotonic_time() delta = now - self.last_rate_log_time _log.info("Processed %s updates from driver " "%.1f/s", self.read_count, 1000.0 / delta) self.last_rate_log_time = now # Wrap the update in an EtcdEvent object so we can dispatch it via the # PathDispatcher. n = EtcdEvent("set" if value is not None else "delete", key, value) self.dispatcher.handle_event(n) def _on_config_loaded_from_driver(self, msg): """ Called when we receive a config loaded message from the driver. This message is expected once per resync, when the config is pre-loaded by the driver. On the first call, responds to the driver synchronously with a config response. If the config has changed since a previous call, triggers Felix to die. """ global_config = msg[MSG_KEY_GLOBAL_CONFIG] host_config = msg[MSG_KEY_HOST_CONFIG] _log.info("Config loaded by driver:\n" "Global: %s\nPer-host: %s", global_config, host_config) if self.configured.is_set(): # We've already been configured. We don't yet support # dynamic config update so instead we check if the config # has changed and die if it has. _log.info("Checking configuration for changes...") if (host_config != self.last_host_config or global_config != self.last_global_config): _log.warning("Felix configuration has changed, " "felix must restart.") _log.info("Old host config: %s", self.last_host_config) _log.info("New host config: %s", host_config) _log.info("Old global config: %s", self.last_global_config) _log.info("New global config: %s", global_config) die_and_restart() else: # First time loading the config. Report it to the config # object. Take copies because report_etcd_config is # destructive. self.last_host_config = host_config.copy() self.last_global_config = global_config.copy() self._config.report_etcd_config(host_config, global_config) # Config now fully resolved, inform the driver. driver_log_file = self._config.DRIVERLOGFILE self._msg_writer.send_message( MSG_TYPE_CONFIG, { MSG_KEY_LOG_FILE: driver_log_file, MSG_KEY_SEV_FILE: self._config.LOGLEVFILE, MSG_KEY_SEV_SCREEN: self._config.LOGLEVSCR, MSG_KEY_SEV_SYSLOG: self._config.LOGLEVSYS, MSG_KEY_PROM_PORT: self._config.PROM_METRICS_DRIVER_PORT if self._config.PROM_METRICS_ENABLED else None }) self.configured.set() def _on_status_from_driver(self, msg): """ Called when we receive a status update from the driver. The driver sends us status messages whenever its status changes. It moves through these states: (1) wait-for-ready (waiting for the global ready flag to become set) (2) resync (resyncing with etcd, processing a snapshot and any concurrent events) (3) in-sync (snapshot processsing complete, now processing only events from etcd) If the driver falls out of sync with etcd then it will start again from (1). If the status is in-sync, triggers the relevant processing. """ status = msg[MSG_KEY_STATUS] _log.info("etcd driver status changed to %s", status) if status == STATUS_IN_SYNC and not self._been_in_sync: # We're now in sync, tell the Actors that need to do start-of-day # cleanup. self.begin_polling.wait() # Make sure splitter is set. self._been_in_sync = True self.splitter.on_datamodel_in_sync() if self._config.REPORT_ENDPOINT_STATUS: self._status_reporter.clean_up_endpoint_statuses(async=True) self._update_hosts_ipset() if self._config.USAGE_REPORT: self._usage_report_greenlet.start() def _start_driver(self): """ Starts the driver subprocess, connects to it over the socket and sends it the init message. Stores the Popen object in self._driver_process for future access. :return: the connected socket to the driver. """ _log.info("Creating server socket.") if os.path.exists("/run"): # Linux FHS version 3.0+ location for runtime sockets etc. sck_filename = "/run/felix-driver.sck" else: # Older Linux versions use /var/run. sck_filename = "/var/run/felix-driver.sck" try: os.unlink(sck_filename) except OSError: _log.debug("Failed to delete driver socket, assuming it " "didn't exist.") update_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) update_socket.bind(sck_filename) update_socket.listen(1) if getattr(sys, "frozen", False): # We're running under pyinstaller, where we share our executable # with the etcd driver. Re-run this executable with the "driver" # argument to invoke the etcd driver. cmd = [sys.argv[0], "driver"] else: # Not running under pyinstaller, execute the etcd driver directly. cmd = [sys.executable, "-m", "calico.etcddriver"] # etcd driver takes the felix socket name as argument. cmd += [sck_filename] _log.info("etcd-driver command line: %s", cmd) self._driver_process = subprocess.Popen(cmd) _log.info("Started etcd driver with PID %s", self._driver_process.pid) with gevent.Timeout(10): update_conn, _ = update_socket.accept() _log.info("Accepted connection on socket") # No longer need the server socket, remove it. try: os.unlink(sck_filename) except OSError: # Unexpected but carry on... _log.exception("Failed to unlink socket") else: _log.info("Unlinked server socket") # Wrap the socket in reader/writer objects that simplify using the # protocol. reader = MessageReader(update_conn) writer = MessageWriter(update_conn) # Give the driver its config. writer.send_message( MSG_TYPE_INIT, { MSG_KEY_ETCD_URLS: [ self._config.ETCD_SCHEME + "://" + addr for addr in self._config.ETCD_ADDRS ], MSG_KEY_HOSTNAME: self._config.HOSTNAME, MSG_KEY_KEY_FILE: self._config.ETCD_KEY_FILE, MSG_KEY_CERT_FILE: self._config.ETCD_CERT_FILE, MSG_KEY_CA_FILE: self._config.ETCD_CA_FILE, }) return reader, writer def on_endpoint_set(self, response, hostname, orchestrator, workload_id, endpoint_id): """Handler for endpoint updates, passes the update to the splitter.""" combined_id = WloadEndpointId(hostname, orchestrator, workload_id, endpoint_id) _log.debug("Endpoint %s updated", combined_id) _stats.increment("Endpoint created/updated") endpoint = parse_endpoint(self._config, combined_id, response.value) self.splitter.on_endpoint_update(combined_id, endpoint) def on_endpoint_delete(self, response, hostname, orchestrator, workload_id, endpoint_id): """Handler for endpoint deleted, passes the update to the splitter.""" combined_id = WloadEndpointId(hostname, orchestrator, workload_id, endpoint_id) _log.debug("Endpoint %s deleted", combined_id) _stats.increment("Endpoint deleted") self.splitter.on_endpoint_update(combined_id, None) def on_host_ep_set(self, response, hostname, endpoint_id): """Handler for create/update of host endpoint.""" combined_id = HostEndpointId(hostname, endpoint_id) _log.debug("Host iface %s updated", combined_id) _stats.increment("Host iface created/updated") iface_data = parse_host_ep(self._config, combined_id, response.value) self.splitter.on_host_ep_update(combined_id, iface_data) def on_host_ep_delete(self, response, hostname, endpoint_id): """Handler for delete of host endpoint.""" combined_id = HostEndpointId(hostname, endpoint_id) _log.debug("Host iface %s deleted", combined_id) _stats.increment("Host iface deleted") self.splitter.on_host_ep_update(combined_id, None) def on_rules_set(self, response, profile_id): """Handler for rules updates, passes the update to the splitter.""" _log.debug("Rules for %s set", profile_id) _stats.increment("Rules created/updated") rules = parse_profile(profile_id, response.value) profile_id = intern(profile_id.encode("utf8")) self.splitter.on_rules_update(profile_id, rules) def on_rules_delete(self, response, profile_id): """Handler for rules deletes, passes the update to the splitter.""" _log.debug("Rules for %s deleted", profile_id) _stats.increment("Rules deleted") self.splitter.on_rules_update(profile_id, None) def on_tags_set(self, response, profile_id): """Handler for tags updates, passes the update to the splitter.""" _log.debug("Tags for %s set", profile_id) _stats.increment("Tags created/updated") rules = parse_tags(profile_id, response.value) profile_id = intern(profile_id.encode("utf8")) self.splitter.on_tags_update(profile_id, rules) def on_tags_delete(self, response, profile_id): """Handler for tags deletes, passes the update to the splitter.""" _log.debug("Tags for %s deleted", profile_id) _stats.increment("Tags deleted") self.splitter.on_tags_update(profile_id, None) def on_prof_labels_set(self, response, profile_id): """Handler for profile labels, passes update to the splitter.""" _log.debug("Labels for profile %s created/updated", profile_id) labels = parse_labels(profile_id, response.value) profile_id = intern(profile_id.encode("utf8")) self.splitter.on_prof_labels_set(profile_id, labels) def on_prof_labels_delete(self, response, profile_id): """Handler for profile label deletion passed update to the splitter.""" _log.debug("Labels for profile %s deleted", profile_id) profile_id = intern(profile_id.encode("utf8")) self.splitter.on_prof_labels_set(profile_id, None) def on_tier_data_set(self, response, tier): _log.debug("Tier data set for tier %s", tier) _stats.increment("Tier data created/updated") data = parse_tier_data(tier, response.value) self.splitter.on_tier_data_update(tier, data) def on_tier_data_delete(self, response, tier): _log.debug("Tier data deleted for tier %s", tier) _stats.increment("Tier data deleted") self.splitter.on_tier_data_update(tier, None) def on_tiered_policy_set(self, response, tier, policy_id): _log.debug("Rules for %s/%s set", tier, policy_id) _stats.increment("Tiered rules created/updated") policy_id = TieredPolicyId(tier, policy_id) rules = parse_policy(policy_id, response.value) if rules is not None: selector = rules.pop("selector") order = rules.pop("order") self.splitter.on_rules_update(policy_id, rules) self.splitter.on_policy_selector_update(policy_id, selector, order) else: self.splitter.on_rules_update(policy_id, None) self.splitter.on_policy_selector_update(policy_id, None, None) def on_tiered_policy_delete(self, response, tier, policy_id): """Handler for tiered rules deletes, passes update to the splitter.""" _log.debug("Rules for %s/%s deleted", tier, policy_id) _stats.increment("tiered rules deleted") policy_id = TieredPolicyId(tier, policy_id) self.splitter.on_rules_update(policy_id, None) self.splitter.on_policy_selector_update(policy_id, None, None) def on_host_ip_set(self, response, hostname): _stats.increment("Host IP created/updated") ip = parse_host_ip(hostname, response.value) if ip: self.ipv4_by_hostname[hostname] = ip else: _log.warning( "Invalid IP for hostname %s: %s, treating as " "deletion", hostname, response.value) self.ipv4_by_hostname.pop(hostname, None) self._update_hosts_ipset() def on_host_ip_delete(self, response, hostname): _stats.increment("Host IP deleted") if self.ipv4_by_hostname.pop(hostname, None): self._update_hosts_ipset() def _update_hosts_ipset(self): if not self._config.IP_IN_IP_ENABLED: _log.debug( "Ignoring update to hosts ipset because IP-in-IP is disabled") return if not self._been_in_sync: _log.debug("Deferring update to hosts ipset until we're in-sync") return self.hosts_ipset.replace_members(frozenset( self.ipv4_by_hostname.values()), async=True) def _on_config_updated(self, response, config_param): new_value = response.value if self.last_global_config.get(config_param) != new_value: _log.critical( "Global config value %s updated. Felix must be " "restarted.", config_param) die_and_restart() _stats.increment("Global config (non) updates") def _on_host_config_updated(self, response, hostname, config_param): if hostname != self._config.HOSTNAME: _log.debug("Ignoring config update for host %s", hostname) return _stats.increment("Per-host config created/updated") new_value = response.value if self.last_host_config.get(config_param) != new_value: _log.critical( "Global config value %s updated. Felix must be " "restarted.", config_param) die_and_restart() def on_ipam_v4_pool_set(self, response, pool_id): _stats.increment("IPAM pool created/updated") pool = parse_ipam_pool(pool_id, response.value) self.splitter.on_ipam_pool_updated(pool_id, pool) def on_ipam_v4_pool_delete(self, response, pool_id): _stats.increment("IPAM pool deleted") self.splitter.on_ipam_pool_updated(pool_id, None)
class AceClient(object): def __init__(self, host, port, connect_timeout=5, result_timeout=10): # Receive buffer self._recvbuffer = None # Stream URL self._url = None # Ace stream socket self._socket = None # Result timeout self._resulttimeout = result_timeout # Shutting down flag self._shuttingDown = Event() # Product key self._product_key = None # Current STATUS self._status = None # Current STATE self._state = None # Current AUTH self._auth = None self._gender = None self._age = None # Result (Created with AsyncResult() on call) self._result = AsyncResult() self._authevent = Event() # Result for getURL() self._urlresult = AsyncResult() # Event for resuming from PAUSE self._resumeevent = Event() # Logger logger = logging.getLogger('AceClient_init') try: self._socket = telnetlib.Telnet(host, port, connect_timeout) logger.info("Successfully connected with Ace!") except Exception as e: raise AceException( "Socket creation error! Ace is not running? " + repr(e)) # Spawning recvData greenlet gevent.spawn(self._recvData) gevent.sleep() def __del__(self): # Destructor just calls destroy() method self.destroy() def destroy(self): ''' AceClient Destructor ''' if self._shuttingDown.isSet(): # Already in the middle of destroying return # Logger logger = logging.getLogger("AceClient_destroy") # We should resume video to prevent read greenlet deadlock self._resumeevent.set() # And to prevent getUrl deadlock self._urlresult.set() # Trying to disconnect try: logger.debug("Destroying client...") self._shuttingDown.set() self._write(AceMessage.request.SHUTDOWN) except: # Ignore exceptions on destroy pass finally: self._shuttingDown.set() def _write(self, message): try: self._socket.write(message + "\r\n") except EOFError as e: raise AceException("Write error! " + repr(e)) def aceInit(self, gender=AceConst.SEX_MALE, age=AceConst.AGE_18_24, product_key=None, pause_delay=0): self._product_key = product_key self._gender = gender self._age = age # PAUSE/RESUME delay self._pausedelay = pause_delay # Logger logger = logging.getLogger("AceClient_aceInit") # Sending HELLO self._write(AceMessage.request.HELLO) if not self._authevent.wait(self._resulttimeout): errmsg = "Authentication timeout. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return if not self._auth: errmsg = "Authentication error. Wrong key?" logger.error(errmsg) raise AceException(errmsg) return logger.debug("aceInit ended") def _getResult(self): # Logger logger = logging.getLogger("AceClient_START") try: result = self._result.get(timeout=self._resulttimeout) if not result: errmsg = "START error!" logger.error(errmsg) raise AceException(errmsg) except gevent.Timeout: errmsg = "START timeout!" logger.error(errmsg) raise AceException(errmsg) return result def START(self, datatype, value): ''' Start video method ''' self._result = AsyncResult() self._urlresult = AsyncResult() self._write(AceMessage.request.LOADASYNC(datatype.upper(), 0, value)) contentinfo = self._getResult() self._write(AceMessage.request.START(datatype.upper(), value)) self._getResult() return contentinfo def getUrl(self, timeout=40): # Logger logger = logging.getLogger("AceClient_getURL") try: res = self._urlresult.get(timeout=timeout) return res except gevent.Timeout: errmsg = "getURL timeout!" logger.error(errmsg) raise AceException(errmsg) def getPlayEvent(self, timeout=None): ''' Blocking while in PAUSE, non-blocking while in RESUME ''' self._resumeevent.wait(timeout=timeout) return def _recvData(self): ''' Data receiver method for greenlet ''' logger = logging.getLogger('AceClient_recvdata') while True: gevent.sleep() try: self._recvbuffer = self._socket.read_until("\r\n") self._recvbuffer = self._recvbuffer.strip() except: # If something happened during read, abandon reader. if not self._shuttingDown.isSet(): logger.error("Exception at socket read") self._shuttingDown.set() return if self._recvbuffer: # Parsing everything only if the string is not empty if self._recvbuffer.startswith(AceMessage.response.HELLO): # Parse HELLO if 'key=' in self._recvbuffer: self._request_key = self._recvbuffer.split()[ 2].split('=')[1] try: self._write(AceMessage.request.READY_key( self._request_key, self._product_key, self._resulttimeout)) except urllib2.URLError as e: logger.error("Can't connect to keygen server! " + \ repr(e)) self._auth = False self._authevent.set() self._request_key = None else: self._write(AceMessage.request.READY_nokey) elif self._recvbuffer.startswith(AceMessage.response.NOTREADY): # NOTREADY logger.error("Ace is not ready. Wrong auth?") self._auth = False self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.LOADRESP): # LOADRESP _contentinfo_raw = self._recvbuffer.split()[2:] _contentinfo_raw = ' '.join(_contentinfo_raw) _contentinfo = json.loads(_contentinfo_raw) if _contentinfo.get('status') == 100: logger.error("LOADASYNC returned error with message: %s" % _contentinfo.get('message')) self._result.set(False) else: logger.debug("Content info: %s", _contentinfo) _filename = urllib2.unquote(_contentinfo.get('files')[0][0]) self._result.set(_filename) elif self._recvbuffer.startswith(AceMessage.response.START): # START try: self._url = self._recvbuffer.split()[1] self._urlresult.set(self._url) self._resumeevent.set() except IndexError as e: self._url = None elif self._recvbuffer.startswith(AceMessage.response.STOP): pass elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN): logger.debug("Got SHUTDOWN from engine") self._socket.close() return elif self._recvbuffer.startswith(AceMessage.response.AUTH): try: self._auth = self._recvbuffer.split()[1] # Send USERDATA here self._write( AceMessage.request.USERDATA(self._gender, self._age)) except: pass self._authevent.set() elif self._recvbuffer.startswith(AceMessage.response.GETUSERDATA): raise AceException("You should init me first!") elif self._recvbuffer.startswith(AceMessage.response.STATE): self._state = self._recvbuffer.split()[1] elif self._recvbuffer.startswith(AceMessage.response.STATUS): self._tempstatus = self._recvbuffer.split()[1].split(';')[0] if self._tempstatus != self._status: self._status = self._tempstatus logger.debug("STATUS changed to " + self._status) if self._status == 'main:err': logger.error( self._status + ' with message ' + self._recvbuffer.split(';')[2]) self._result.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) self._urlresult.set_exception( AceException(self._status + ' with message ' + self._recvbuffer.split(';')[2])) elif self._status == 'main:starting': self._result.set(True) elif self._recvbuffer.startswith(AceMessage.response.PAUSE): logger.debug("PAUSE event") self._resumeevent.clear() elif self._recvbuffer.startswith(AceMessage.response.RESUME): logger.debug("RESUME event") gevent.sleep(self._pausedelay) self._resumeevent.set()