def __init__(self, client_id, client_secret, endpoints=None, name='tvm'): """TVM :param client_id: Integer client identifier. :param client_secret: Client secret. :param endpoints: TVM service endpoints list in format (host, port). :param name: TVM service name, defaults to 'tvm'. """ self._client_id = client_id self._client_secret = client_secret self._tvm = Service(name, endpoints) if endpoints else Service(name)
def get_info(flags): node = Service('node') ch = yield node.info('ppn', flags) info = yield ch.rx.get() raise gen.Return(info)
def app_ctl(): apps = [ Service('Echo'), Service('ppn'), # Service('Echo1'), # Service('Echo2'), # Service('Echo3'), # Service('Echo4'), # Service('Echo5'), ] x = 0 while True: yield [control_app(app, x + i) for i, app in enumerate(apps)] x += 0.02 break
def get_secure_unicorn(): config = Config(SharedStatus()) config.update() return SecureServiceFabric.make_secure_adaptor( Service('unicorn'), *config.secure, endpoints=config.locator_endpoints)
def loop(): node = Service('node') apps = ['Echo{}'.format(i) for i in xrange(1, 2)] channels = dict() #for a in apps: # print ('registering app {}'.format(a)) # channels[a] = yield node.control(a) ch = yield node.control('Echo1') print('before seq') #yield [ch.tx.write(sin_square(AMPH, t * 10)) for t in xrange(1,10)] print('DONE after seq') x = 0.0 while True: try: print('Sending control.') # yield [channels[a].tx.write(sin_square(AMPH, x)) for a in apps] yield ch.tx.write(sin_square(AMPH, 10)) except Exception as e: print 'error {}'.format(e) yield gen.sleep(ERR_SLEEP) ch = yield node.control('Echo1') else: print 'control is done' yield gen.sleep(LOOP_TO) x += DELTA
def main(): storage = Service("storage", endpoints=[["localhost", 10053]], io_loop=io) channel = yield storage.find('app', ['apps']) res = yield channel.rx.get() raise gen.Return(res)
def test_node_service_bad_on_read(): io = IOLoop.current() node = Service("node", endpoints=[["localhost", 10053]], io_loop=io) malformed_message = msgpack.packb([-999, 0]) node.on_read(malformed_message) message = msgpack.packb([-999, 0, []]) node.on_read(message)
def wrapper(): active_apps = len(self.cache[name]) self.logger.info( "%s: preparing to moving %s %s to an inactive queue (active %d)", app.id, app.name, "{0}:{1}".format(*app.address), active_apps) try: new_app = Service(name, locator=self.locator, timeout=RESOLVE_TIMEOUT) self.logger.info("%s: creating an instance of %s", new_app.id, name) yield new_app.connect() self.logger.info("%s: connect to an app %s endpoint %s ", new_app.id, new_app.name, "{0}:{1}".format(*new_app.address)) timeout = (1 + random.random()) * self.refresh_period self.io_loop.call_later(timeout, self.move_to_inactive(new_app, name)) # add to cache only after successfully connected self.cache[name].append(new_app) except Exception as err: self.logger.error("%s: unable to connect to `%s`: %s", new_app.id, name, err) # schedule later self.io_loop.call_later(self.get_timeout(name), self.move_to_inactive(app, name)) else: self.logger.info("%s: move %s %s to an inactive queue", app.id, app.name, "{0}:{1}".format(*app.address)) # current active app will be dropped here self.migrate_from_cache_to_inactive(app, name)
def test_service_invalid_api_version(): io = IOLoop.current() storage = Service("storage", endpoints=[["localhost", 10053]], version=100, io_loop=io) io.run_sync(storage.connect)
def stop_app(self, appname): succeed = list() failed = list() hosts_count = len(self.hosts) for i, host in enumerate(self.hosts): log.info("Stop %s at host %d/%d %s" % (appname, i, hosts_count, host)) nodeinstance = None try: nodeinstance = Service("node", blockingConnect=False) yield nodeinstance.connect(host=host) res = yield app.Stop(nodeinstance, appname).execute() self.logcallback(str(res) + '\n') except Exception as e: item = "Unable to connect to node at host %s %s\n" % (host, e) log.error(item) self.logcallback(item) failed.append(host) else: item = "App %s has been stoped successfully\n" % appname log.info(item) self.logcallback(item) succeed.append(host) finally: if nodeinstance is not None: nodeinstance.disconnect() yield (succeed, failed)
def main( uuid_prefix, uniresis_stub_uuid, to_sleep, state_file, verify_url, max_workers, proportion): config = Config(SharedStatus()) config.update() def load_state(fname): print('reading state for emulation from {}'.format(fname)) with open(fname) as fl: return yaml.load(fl) emul_state = load_state(state_file) if state_file else dict( ppn=[(DEFAULT_PROFILE1, 100), ], Echo=[(DEFAULT_PROFILE1, 50), (DEFAULT_PROFILE2, 5)], EchoWeb=[(DEFAULT_PROFILE1, 50), (DEFAULT_PROFILE2, 10)], ) # TODO: not yet implemented (released actually) in framework! unicorn = SecureServiceFabric.make_secure_adaptor( Service('unicorn'), *config.secure, endpoints=config.locator_endpoints) IOLoop.current().run_sync( lambda: state_pusher( unicorn, uuid_prefix, uniresis_stub_uuid, emul_state, max_workers, to_sleep, verify_url, proportion))
def ping_loop(): echo = Service(DEFAULT_SERVICE) ch = yield echo.enqueue('version') version = yield ch.rx.get(timeout=5) print 'version: {}'.format(version) cnt = 1 while True: msg = '{}_{}'.format(DEFAULT_MESSAGE, cnt) cnt += 1 # msg = msgpack.packb(msg) print 'sending ping message {}'.format(msg) ch = yield echo.enqueue('ping') _ = yield ch.tx.write(msg) answer = yield ch.rx.get(timeout=10) print 'ans {}'.format(answer) yield gen.sleep(DEFAULT_SLEEP)
def loop(): node = Service('node') apps = [ ('Echo1', 'IsoProcess'), ('Echo2', 'IsoProcess'), ('Echo3', 'IsoProcess'), ('Echo4', 'IsoProcess'), # from unsable hosts #('faces:1', 'vision'), #('licenseplate:8', 'vision'), #('superres:2', 'vision'), ] i = 0 z = 0 while True: i ^= 1 z += 1 if i: yield [start(node, app, profile) for app, profile in apps] chs = yield [make_ch(node, app) for app, _ in apps] for y in xrange(10): yield [control(c, y + z) for c in chs] yield gen.sleep(1) yield [close_ch(c) for c in chs] else: yield [stop(node, app) for app, _ in apps] yield gen.sleep(10)
def _get(self, app): now = time.time() app_service = Service(app) \ if app not in self.apps else self.apps[app].application self.apps[app] = _AppsCache.Record(app_service, now) return app_service
def get_service(self, name, request): # cache isn't full for the current application if len(self.cache[name]) < self.spool_size: logger = request.logger try: app = Service(name, locator=self.locator, timeout=RESOLVE_TIMEOUT) logger.info("%s: creating an instance of %s", app.id, name) self.cache[name].append(app) yield app.connect(request.traceid) logger.info("%s: connect to an app %s endpoint %s ", app.id, app.name, "{0}:{1}".format(*app.address)) timeout = (1 + random.random()) * self.refresh_period self.io_loop.call_later(timeout, self.move_to_inactive(app, name)) except Exception as err: logger.error("%s: unable to connect to `%s`: %s", app.id, name, err) drop_app_from_cache(self.cache, app, name) raise gen.Return() else: raise gen.Return(app) # get an instance from cache chosen = random.choice(self.cache[name]) raise gen.Return(chosen)
def reelect_app(self, request, app): """tries to connect to the same app on differnet host from dist-info""" # disconnect app explicitly to break possibly existing connection app.disconnect() endpoints_size = len(app.locator.endpoints) # try x times, where x is the number of different endpoints in app locator. for _ in xrange(0, endpoints_size + 1): # last chance to take app from common pool if len(app.locator.endpoints) == 0: request.logger.info( "giving up on connecting to dist-info hosts, falling back to common pool processing" ) app = yield self.proxy.reelect_app(request, app) raise gen.Return(app) try: # always create new locator to prevent locking as we do connect with timeout # however lock can be still held during TCP timeout locator = Locator(endpoints=app.locator.endpoints) request.logger.info("connecting to locator %s", locator.endpoints[0]) # first try to connect to locator only on remote host with timeout yield gen.with_timeout(self.service_connect_timeout, locator.connect()) request.logger.debug("connected to locator %s for %s", locator.endpoints[0], app.name) app = Service(app.name, locator=locator, timeout=RESOLVE_TIMEOUT) # try to resolve and connect to application itself yield gen.with_timeout(self.service_connect_timeout, app.connect()) request.logger.debug("connected to application %s via %s", app.name, app.endpoints) except gen.TimeoutError: # on timeout try next endpoint first request.logger.warning( "timed out while connecting to application") continue except ServiceError as err: request.logger.warning("got error while resolving app - %s", err) if err.category in LOCATORCATEGORY and err.code == ESERVICENOTAVAILABLE: # if the application is down - also try next endpoint continue else: raise err finally: # drop first endpoint to start next connection from different endpoint # we do this, as default logic of connection attempts in locator do not fit here app.locator.endpoints = app.locator.endpoints[1:] # return connected app raise gen.Return(app) raise PluginApplicationError(42, 42, "could not connect to application")
def put(self, name): try: s = Service(name) except Exception: return None else: self._cache[name] = s return s
def control(times): node = Service('node') logger = Service('logging') try: ch = yield node.start_app(APP_NAME, PROFILE) result = yield ch.rx.get() print('start res {}'.format(result)) yield logger.emit except Exception: pass control_channel = yield node.control(APP_NAME) for i in xrange(0, times): # print('running {}'.format(i)) yield control_channel.tx.write(i % 10)
def create_service(self, name): if name not in self._cache: if name == 'locator': service = Locator(endpoints=self._endpoints) else: service = Service(name, endpoints=self._endpoints) self._cache[name] = service return self._cache[name]
def instance(host="localhost", port=10053): if not hasattr(FlowTools, "_instance"): with FlowTools._instance_lock: if not hasattr(FlowTools, "_instance"): from cocaine.services import Service FlowTools._instance = Service("flow-tools", host=host, port=port) return FlowTools._instance
def connect(self, host='localhost', port=10053): try: self.storage = Service('storage', host, port) except socket.error as err: if err.errno == errno.ECONNREFUSED: raise ConnectionRefusedError((host, port)) else: raise ConnectionError( (host, port), 'Unknown connection error: {0}'.format(err))
def f(): io = IOLoop.current() storage = Service("storage", endpoints=[["localhost", 10053]], io_loop=io) channel = yield storage.find('app', ['apps']) app_list = yield channel.rx.get() assert isinstance(app_list, list) raise gen.Return("OK")
def main(cmd, path, subscribers, state_size, times): unicorn = Service('unicorn') if cmd == 'prod': IOLoop.current().run_sync( lambda: do_prod(unicorn, path, state_size, times)) elif cmd == 'subs': IOLoop.current().run_sync( lambda: do_subscribe(unicorn, path, subscribers)) else: click.secho('unknown command {}'.format(cmd), fg='red')
def test_ref_count_of_service(): s = Service("storage") ws = weakref.ref(s) io.run_sync(s.connect, timeout=2) wpipe = weakref.ref(s.pipe) fd = wpipe().fileno().fileno() s = None gc.collect() # there should be no referres to the service assert ws() is None, gc.get_referrers(ws()) assert fd not in io._handlers, "%d %s" % (fd, io._handlers)
def __init__(self): self.storage = Service("storage") self.good_path = os.path.join( os.path.abspath(os.path.dirname(__file__)), "fixtures/docker_app") self.broken_path = os.path.join( os.path.abspath(os.path.dirname(__file__)), "fixtures/broken_docker_app") self.docker_address = os.getenv("DOCKER_HOST") self.registry_address = os.getenv("DOCKER_REGISTRY") if not (self.docker_address and self.registry_address): raise SkipTest("Can't do it without Docker or Registry") self.client = docker.Client(self.docker_address, io_loop=io)
def __init__(self, logging, node): logging.info("Created NodeInfoUpdater") self.__logging = logging self.__node = node self.__tq = timed_queue.TimedQueue() self.__tq.start() self.__session = elliptics.Session(self.__node) self.__session.set_timeout(config.get('wait_timeout', 5)) self.__nodeUpdateTimestamps = (time.time(), time.time()) self.__cache = Service('cache') self.loadNodes(delayed=False)
def delete_node(node): unicorn = Service('unicorn') ch = yield unicorn.get(node) _, version = yield ch.rx.get() if version == -1: return else: ch = yield unicorn.remove(node, version) yield ch.rx.get()
def get_service(self, name): if len(cache[name]) < self.spoolSize - len(dying[name]): try: created = [] for _ in xrange(self.spoolSize - len(cache[name])): app = Service(name) created.append(app) self.logger.info("Connect to app: %s endpoint %s ", app.name, "{0}:{1}".format(*app.address)) cache[name].extend(created) for app in created: timeout = (1 + random.random()) * self.refreshPeriod self.io_loop.add_timeout(time.time() + timeout, self.move_to_inactive(app, name)) except Exception as err: self.logger.error(str(err)) return None chosen = random.choice(cache[name]) if chosen.isConnected(): try: fd = chosen._pipe.sock.fileno() chosen._ioLoop._fd_events[fd] except Exception as err: self.logger.exception( "Wrong fd or missing poll event, so remove this service") cache[name].remove(chosen) return self.get_service(name) return chosen else: self.logger.warning("Service %s disconnected %s", chosen.name, chosen.address) try: chosen.reconnect(blocking=True) if chosen.isConnected(): try: fd = chosen._pipe.sock.fileno() chosen._ioLoop._fd_events[fd] except Exception as err: self.logger.exception( "Wrong fd or missing poll event, so remove this service" ) cache[name].remove(chosen) return self.get_service(name) self.logger.info("Service %s has reconnected successfully", chosen.name) return chosen else: return None except Exception as err: return None
def add_node(node, val): unicorn = Service('unicorn') ch = yield unicorn.get(node) _, version = yield ch.rx.get() if version == -1: ch = yield unicorn.create(node, val) yield ch.rx.get() else: print 'puting data {}'.format(val) ch = yield unicorn.put(node, val, version) yield ch.rx.get()
def get_service_with_seed(self, name, seed, request): logger = request.logger app = Service(name, seed=seed, locator=self.locator) try: logger.info("%s: creating an instance of %s, seed %s", app.id, name, seed) yield app.connect(request.traceid) except Exception as err: logger.error("%s: unable to connect to `%s`: %s", app.id, name, err) raise gen.Return() raise gen.Return(app)