def __init__(self, client_id, client_secret, endpoints=None, name='tvm'):
"""TVM
:param client_id: Integer client identifier.
:param client_secret: Client secret.
:param endpoints: TVM service endpoints list in format (host, port).
:param name: TVM service name, defaults to 'tvm'.
"""
self._client_id = client_id
self._client_secret = client_secret
self._tvm = Service(name, endpoints) if endpoints else Service(name)
def get_info(flags):
node = Service('node')
ch = yield node.info('ppn', flags)
info = yield ch.rx.get()
raise gen.Return(info)
def app_ctl():
apps = [
Service('Echo'),
Service('ppn'),
# Service('Echo1'),
# Service('Echo2'),
# Service('Echo3'),
# Service('Echo4'),
# Service('Echo5'),
]
x = 0
while True:
yield [control_app(app, x + i) for i, app in enumerate(apps)]
x += 0.02
break
def get_secure_unicorn():
config = Config(SharedStatus())
config.update()
return SecureServiceFabric.make_secure_adaptor(
Service('unicorn'), *config.secure, endpoints=config.locator_endpoints)
def loop():
node = Service('node')
apps = ['Echo{}'.format(i) for i in xrange(1, 2)]
channels = dict()
#for a in apps:
# print ('registering app {}'.format(a))
# channels[a] = yield node.control(a)
ch = yield node.control('Echo1')
print('before seq')
#yield [ch.tx.write(sin_square(AMPH, t * 10)) for t in xrange(1,10)]
print('DONE after seq')
x = 0.0
while True:
try:
print('Sending control.')
# yield [channels[a].tx.write(sin_square(AMPH, x)) for a in apps]
yield ch.tx.write(sin_square(AMPH, 10))
except Exception as e:
print 'error {}'.format(e)
yield gen.sleep(ERR_SLEEP)
ch = yield node.control('Echo1')
else:
print 'control is done'
yield gen.sleep(LOOP_TO)
x += DELTA
def main():
storage = Service("storage",
endpoints=[["localhost", 10053]],
io_loop=io)
channel = yield storage.find('app', ['apps'])
res = yield channel.rx.get()
raise gen.Return(res)
def test_node_service_bad_on_read():
io = IOLoop.current()
node = Service("node", endpoints=[["localhost", 10053]], io_loop=io)
malformed_message = msgpack.packb([-999, 0])
node.on_read(malformed_message)
message = msgpack.packb([-999, 0, []])
node.on_read(message)
def wrapper():
active_apps = len(self.cache[name])
self.logger.info(
"%s: preparing to moving %s %s to an inactive queue (active %d)",
app.id, app.name, "{0}:{1}".format(*app.address), active_apps)
try:
new_app = Service(name,
locator=self.locator,
timeout=RESOLVE_TIMEOUT)
self.logger.info("%s: creating an instance of %s", new_app.id,
name)
yield new_app.connect()
self.logger.info("%s: connect to an app %s endpoint %s ",
new_app.id, new_app.name,
"{0}:{1}".format(*new_app.address))
timeout = (1 + random.random()) * self.refresh_period
self.io_loop.call_later(timeout,
self.move_to_inactive(new_app, name))
# add to cache only after successfully connected
self.cache[name].append(new_app)
except Exception as err:
self.logger.error("%s: unable to connect to `%s`: %s",
new_app.id, name, err)
# schedule later
self.io_loop.call_later(self.get_timeout(name),
self.move_to_inactive(app, name))
else:
self.logger.info("%s: move %s %s to an inactive queue", app.id,
app.name, "{0}:{1}".format(*app.address))
# current active app will be dropped here
self.migrate_from_cache_to_inactive(app, name)
def test_service_invalid_api_version():
io = IOLoop.current()
storage = Service("storage",
endpoints=[["localhost", 10053]],
version=100,
io_loop=io)
io.run_sync(storage.connect)
def stop_app(self, appname):
succeed = list()
failed = list()
hosts_count = len(self.hosts)
for i, host in enumerate(self.hosts):
log.info("Stop %s at host %d/%d %s" %
(appname, i, hosts_count, host))
nodeinstance = None
try:
nodeinstance = Service("node", blockingConnect=False)
yield nodeinstance.connect(host=host)
res = yield app.Stop(nodeinstance, appname).execute()
self.logcallback(str(res) + '\n')
except Exception as e:
item = "Unable to connect to node at host %s %s\n" % (host, e)
log.error(item)
self.logcallback(item)
failed.append(host)
else:
item = "App %s has been stoped successfully\n" % appname
log.info(item)
self.logcallback(item)
succeed.append(host)
finally:
if nodeinstance is not None:
nodeinstance.disconnect()
yield (succeed, failed)
def main(
uuid_prefix, uniresis_stub_uuid, to_sleep, state_file, verify_url,
max_workers, proportion):
config = Config(SharedStatus())
config.update()
def load_state(fname):
print('reading state for emulation from {}'.format(fname))
with open(fname) as fl:
return yaml.load(fl)
emul_state = load_state(state_file) if state_file else dict(
ppn=[(DEFAULT_PROFILE1, 100), ],
Echo=[(DEFAULT_PROFILE1, 50), (DEFAULT_PROFILE2, 5)],
EchoWeb=[(DEFAULT_PROFILE1, 50), (DEFAULT_PROFILE2, 10)],
)
# TODO: not yet implemented (released actually) in framework!
unicorn = SecureServiceFabric.make_secure_adaptor(
Service('unicorn'), *config.secure, endpoints=config.locator_endpoints)
IOLoop.current().run_sync(
lambda:
state_pusher(
unicorn, uuid_prefix, uniresis_stub_uuid, emul_state,
max_workers, to_sleep, verify_url, proportion))
def ping_loop():
echo = Service(DEFAULT_SERVICE)
ch = yield echo.enqueue('version')
version = yield ch.rx.get(timeout=5)
print 'version: {}'.format(version)
cnt = 1
while True:
msg = '{}_{}'.format(DEFAULT_MESSAGE, cnt)
cnt += 1
# msg = msgpack.packb(msg)
print 'sending ping message {}'.format(msg)
ch = yield echo.enqueue('ping')
_ = yield ch.tx.write(msg)
answer = yield ch.rx.get(timeout=10)
print 'ans {}'.format(answer)
yield gen.sleep(DEFAULT_SLEEP)
def loop():
node = Service('node')
apps = [
('Echo1', 'IsoProcess'),
('Echo2', 'IsoProcess'),
('Echo3', 'IsoProcess'),
('Echo4', 'IsoProcess'),
# from unsable hosts
#('faces:1', 'vision'),
#('licenseplate:8', 'vision'),
#('superres:2', 'vision'),
]
i = 0
z = 0
while True:
i ^= 1
z += 1
if i:
yield [start(node, app, profile) for app, profile in apps]
chs = yield [make_ch(node, app) for app, _ in apps]
for y in xrange(10):
yield [control(c, y + z) for c in chs]
yield gen.sleep(1)
yield [close_ch(c) for c in chs]
else:
yield [stop(node, app) for app, _ in apps]
yield gen.sleep(10)
def _get(self, app):
now = time.time()
app_service = Service(app) \
if app not in self.apps else self.apps[app].application
self.apps[app] = _AppsCache.Record(app_service, now)
return app_service
def get_service(self, name, request):
# cache isn't full for the current application
if len(self.cache[name]) < self.spool_size:
logger = request.logger
try:
app = Service(name,
locator=self.locator,
timeout=RESOLVE_TIMEOUT)
logger.info("%s: creating an instance of %s", app.id, name)
self.cache[name].append(app)
yield app.connect(request.traceid)
logger.info("%s: connect to an app %s endpoint %s ", app.id,
app.name, "{0}:{1}".format(*app.address))
timeout = (1 + random.random()) * self.refresh_period
self.io_loop.call_later(timeout,
self.move_to_inactive(app, name))
except Exception as err:
logger.error("%s: unable to connect to `%s`: %s", app.id, name,
err)
drop_app_from_cache(self.cache, app, name)
raise gen.Return()
else:
raise gen.Return(app)
# get an instance from cache
chosen = random.choice(self.cache[name])
raise gen.Return(chosen)
def reelect_app(self, request, app):
"""tries to connect to the same app on differnet host from dist-info"""
# disconnect app explicitly to break possibly existing connection
app.disconnect()
endpoints_size = len(app.locator.endpoints)
# try x times, where x is the number of different endpoints in app locator.
for _ in xrange(0, endpoints_size + 1):
# last chance to take app from common pool
if len(app.locator.endpoints) == 0:
request.logger.info(
"giving up on connecting to dist-info hosts, falling back to common pool processing"
)
app = yield self.proxy.reelect_app(request, app)
raise gen.Return(app)
try:
# always create new locator to prevent locking as we do connect with timeout
# however lock can be still held during TCP timeout
locator = Locator(endpoints=app.locator.endpoints)
request.logger.info("connecting to locator %s",
locator.endpoints[0])
# first try to connect to locator only on remote host with timeout
yield gen.with_timeout(self.service_connect_timeout,
locator.connect())
request.logger.debug("connected to locator %s for %s",
locator.endpoints[0], app.name)
app = Service(app.name,
locator=locator,
timeout=RESOLVE_TIMEOUT)
# try to resolve and connect to application itself
yield gen.with_timeout(self.service_connect_timeout,
app.connect())
request.logger.debug("connected to application %s via %s",
app.name, app.endpoints)
except gen.TimeoutError:
# on timeout try next endpoint first
request.logger.warning(
"timed out while connecting to application")
continue
except ServiceError as err:
request.logger.warning("got error while resolving app - %s",
err)
if err.category in LOCATORCATEGORY and err.code == ESERVICENOTAVAILABLE:
# if the application is down - also try next endpoint
continue
else:
raise err
finally:
# drop first endpoint to start next connection from different endpoint
# we do this, as default logic of connection attempts in locator do not fit here
app.locator.endpoints = app.locator.endpoints[1:]
# return connected app
raise gen.Return(app)
raise PluginApplicationError(42, 42,
"could not connect to application")
def put(self, name):
try:
s = Service(name)
except Exception:
return None
else:
self._cache[name] = s
return s
def control(times):
node = Service('node')
logger = Service('logging')
try:
ch = yield node.start_app(APP_NAME, PROFILE)
result = yield ch.rx.get()
print('start res {}'.format(result))
yield logger.emit
except Exception:
pass
control_channel = yield node.control(APP_NAME)
for i in xrange(0, times):
# print('running {}'.format(i))
yield control_channel.tx.write(i % 10)
def create_service(self, name):
if name not in self._cache:
if name == 'locator':
service = Locator(endpoints=self._endpoints)
else:
service = Service(name, endpoints=self._endpoints)
self._cache[name] = service
return self._cache[name]
def instance(host="localhost", port=10053):
if not hasattr(FlowTools, "_instance"):
with FlowTools._instance_lock:
if not hasattr(FlowTools, "_instance"):
from cocaine.services import Service
FlowTools._instance = Service("flow-tools",
host=host,
port=port)
return FlowTools._instance
def connect(self, host='localhost', port=10053):
try:
self.storage = Service('storage', host, port)
except socket.error as err:
if err.errno == errno.ECONNREFUSED:
raise ConnectionRefusedError((host, port))
else:
raise ConnectionError(
(host, port), 'Unknown connection error: {0}'.format(err))
def f():
io = IOLoop.current()
storage = Service("storage",
endpoints=[["localhost", 10053]],
io_loop=io)
channel = yield storage.find('app', ['apps'])
app_list = yield channel.rx.get()
assert isinstance(app_list, list)
raise gen.Return("OK")
def main(cmd, path, subscribers, state_size, times):
unicorn = Service('unicorn')
if cmd == 'prod':
IOLoop.current().run_sync(
lambda: do_prod(unicorn, path, state_size, times))
elif cmd == 'subs':
IOLoop.current().run_sync(
lambda: do_subscribe(unicorn, path, subscribers))
else:
click.secho('unknown command {}'.format(cmd), fg='red')
def test_ref_count_of_service():
s = Service("storage")
ws = weakref.ref(s)
io.run_sync(s.connect, timeout=2)
wpipe = weakref.ref(s.pipe)
fd = wpipe().fileno().fileno()
s = None
gc.collect()
# there should be no referres to the service
assert ws() is None, gc.get_referrers(ws())
assert fd not in io._handlers, "%d %s" % (fd, io._handlers)
def __init__(self):
self.storage = Service("storage")
self.good_path = os.path.join(
os.path.abspath(os.path.dirname(__file__)), "fixtures/docker_app")
self.broken_path = os.path.join(
os.path.abspath(os.path.dirname(__file__)),
"fixtures/broken_docker_app")
self.docker_address = os.getenv("DOCKER_HOST")
self.registry_address = os.getenv("DOCKER_REGISTRY")
if not (self.docker_address and self.registry_address):
raise SkipTest("Can't do it without Docker or Registry")
self.client = docker.Client(self.docker_address, io_loop=io)
def __init__(self, logging, node):
logging.info("Created NodeInfoUpdater")
self.__logging = logging
self.__node = node
self.__tq = timed_queue.TimedQueue()
self.__tq.start()
self.__session = elliptics.Session(self.__node)
self.__session.set_timeout(config.get('wait_timeout', 5))
self.__nodeUpdateTimestamps = (time.time(), time.time())
self.__cache = Service('cache')
self.loadNodes(delayed=False)
def delete_node(node):
unicorn = Service('unicorn')
ch = yield unicorn.get(node)
_, version = yield ch.rx.get()
if version == -1:
return
else:
ch = yield unicorn.remove(node, version)
yield ch.rx.get()
def get_service(self, name):
if len(cache[name]) < self.spoolSize - len(dying[name]):
try:
created = []
for _ in xrange(self.spoolSize - len(cache[name])):
app = Service(name)
created.append(app)
self.logger.info("Connect to app: %s endpoint %s ",
app.name, "{0}:{1}".format(*app.address))
cache[name].extend(created)
for app in created:
timeout = (1 + random.random()) * self.refreshPeriod
self.io_loop.add_timeout(time.time() + timeout,
self.move_to_inactive(app, name))
except Exception as err:
self.logger.error(str(err))
return None
chosen = random.choice(cache[name])
if chosen.isConnected():
try:
fd = chosen._pipe.sock.fileno()
chosen._ioLoop._fd_events[fd]
except Exception as err:
self.logger.exception(
"Wrong fd or missing poll event, so remove this service")
cache[name].remove(chosen)
return self.get_service(name)
return chosen
else:
self.logger.warning("Service %s disconnected %s", chosen.name,
chosen.address)
try:
chosen.reconnect(blocking=True)
if chosen.isConnected():
try:
fd = chosen._pipe.sock.fileno()
chosen._ioLoop._fd_events[fd]
except Exception as err:
self.logger.exception(
"Wrong fd or missing poll event, so remove this service"
)
cache[name].remove(chosen)
return self.get_service(name)
self.logger.info("Service %s has reconnected successfully",
chosen.name)
return chosen
else:
return None
except Exception as err:
return None
def add_node(node, val):
unicorn = Service('unicorn')
ch = yield unicorn.get(node)
_, version = yield ch.rx.get()
if version == -1:
ch = yield unicorn.create(node, val)
yield ch.rx.get()
else:
print 'puting data {}'.format(val)
ch = yield unicorn.put(node, val, version)
yield ch.rx.get()
def get_service_with_seed(self, name, seed, request):
logger = request.logger
app = Service(name, seed=seed, locator=self.locator)
try:
logger.info("%s: creating an instance of %s, seed %s", app.id,
name, seed)
yield app.connect(request.traceid)
except Exception as err:
logger.error("%s: unable to connect to `%s`: %s", app.id, name,
err)
raise gen.Return()
raise gen.Return(app)
