def test_heartbeat_failure(self):
self.patch_cfg('pyon.ion.process.CFG', {'cc':{'timeout':{'heartbeat_proc_count_threshold':2, 'heartbeat':1.0}}})
svc = self.container.proc_manager.procs[self.pid]
ip = svc._process
stopar = AsyncResult()
self.container.proc_manager.add_proc_state_changed_callback(lambda *args: stopar.set(args))
noticear = AsyncResult() # notify us when the call has been made
ar = ip._routing_call(svc.takes_too_long, None, noticear=noticear)
noticear.get(timeout=10) # wait for the call to be made
# heartbeat a few times so we trigger the failure soon
for x in xrange(2):
ip.heartbeat()
# wait for ip thread to kick over
ip._ctrl_thread.join(timeout=5)
# now wait for notice proc got canned
stopargs = stopar.get(timeout=5)
self.assertEquals(stopargs, (svc, ProcessStateEnum.FAILED, self.container))
# should've shut down, no longer in container's process list
self.assertEquals(len(self.container.proc_manager.procs), 0)
def test_zero_max_size(self):
q = queue.Channel()
def sender(evt, q):
q.put('hi')
evt.set('done')
def receiver(evt, q):
x = q.get()
evt.set(x)
e1 = AsyncResult()
e2 = AsyncResult()
p1 = gevent.spawn(sender, e1, q)
gevent.sleep(0.001)
self.assert_(not e1.ready())
p2 = gevent.spawn(receiver, e2, q)
self.assertEquals(e2.get(), 'hi')
self.assertEquals(e1.get(), 'done')
timeout = gevent.Timeout.start_new(0)
try:
gevent.joinall([p1, p2])
finally:
timeout.cancel()
def test__control_flow_cancelled_call(self):
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
# put a call in that will never finish
waitar = AsyncResult() # test specific, wait for this to indicate we're being processed/hung
callar = AsyncResult() # test specific, an ar that is just waited on by the spin call (eventually set in this test)
def spin(inar, outar):
outar.set(True)
inar.wait()
ar = p._routing_call(spin, MagicMock(), callar, waitar)
# schedule a second call that we're going to cancel
futurear = AsyncResult()
ar2 = p._routing_call(futurear.set, MagicMock(), sentinel.val)
# wait until we get notice we're being processed
waitar.get(timeout=2)
# cancel the SECOND call
p.cancel_or_abort_call(ar2)
# prove we didn't interrupt the current proc by allowing it to continue
callar.set()
ar.get(timeout=2)
# now the second proc will get queued and never called because it is cancelled
self.assertRaises(Timeout, futurear.get, timeout=2)
self.assertTrue(ar2.ready())
def test__control_flow_expired_call(self):
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
def make_call(call, ctx, val):
ar = p._routing_call(call, ctx, val)
return ar.get(timeout=10)
ctx = { 'reply-by' : 0 } # no need for real time, as it compares by CURRENT >= this value
futurear = AsyncResult()
with patch('pyon.ion.process.greenlet') as gcm:
waitar = AsyncResult()
gcm.getcurrent().kill.side_effect = lambda *a, **k: waitar.set()
ar = p._routing_call(futurear.set, ctx, sentinel.val)
waitar.get(timeout=10)
# futurear is not set
self.assertFalse(futurear.ready())
# neither is the ar we got back from routing_call
self.assertFalse(ar.ready())
# we should've been killed, though
self.assertEquals(gcm.getcurrent().kill.call_count, 1)
self.assertIsInstance(gcm.getcurrent().kill.call_args[1]['exception'], IonTimeout)
# put a new call through (to show unblocked)
futurear2 = AsyncResult()
ar2 = p._routing_call(futurear2.set, MagicMock(), sentinel.val2)
ar2.get(timeout=2)
def test__interrupt_control_thread(self):
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
# put a call in that will never finish
waitar = AsyncResult() # test specific, wait for this to indicate we're being processed/hung
callar = AsyncResult() # test specific, an ar that is just waited on by the spin call
def spin(inar, outar):
outar.set(True)
inar.wait()
ar = p._routing_call(spin, MagicMock(), callar, waitar)
# wait until we get notice we're being processed
waitar.get(timeout=2)
# interrupt it
p._interrupt_control_thread()
# the ar we got back from routing_call will not be set, it never finished the call
self.assertFalse(ar.ready())
# to prove we're unblocked, run another call through the control thread
ar2 = p._routing_call(callar.set, MagicMock(), sentinel.val)
ar2.get(timeout=2)
self.assertTrue(callar.ready())
self.assertEquals(callar.get(), sentinel.val)
def test_quit_stops_timers(self):
ar = AsyncResult()
def cb(*args, **kwargs):
ar.set(args)
self.interval_timer_count += 1
event_origin = "test_quitter"
sub = EventSubscriber(event_type="TimerEvent", callback=cb, origin=event_origin)
sub.start()
self.addCleanup(sub.stop)
tid = self.ssclient.create_interval_timer(start_time="now",
interval=1,
event_origin=event_origin)
# wait until at least one scheduled message
ar.get(timeout=5)
# shut it down!
p = self.container.proc_manager.procs_by_name['scheduler']
self.container.terminate_process(p.id)
# assert empty
self.assertEquals(p.schedule_entries, {})
def test_result_cb_error(self):
result = AsyncResult()
DNSResolver._result_cb(result, 13, pycares.errno.ARES_ENOTFOUND)
with self.assertRaises(DNSError) as cm:
result.get()
self.assertEqual('Domain name not found [ARES_ENOTFOUND]',
str(cm.exception))
def test_two_waiters_one_dies(self):
def waiter(q, evt):
evt.set(q.get())
def do_receive(q, evt):
timeout = gevent.Timeout.start_new(0, RuntimeError())
try:
try:
result = q.get()
evt.set(result)
except RuntimeError:
evt.set('timed out')
finally:
timeout.cancel()
q = queue.Queue()
dying_evt = AsyncResult()
waiting_evt = AsyncResult()
gevent.spawn(do_receive, q, dying_evt)
gevent.spawn(waiter, q, waiting_evt)
gevent.sleep(0.01)
q.put('hi')
self.assertEquals(dying_evt.get(), 'timed out')
self.assertEquals(waiting_evt.get(), 'hi')
class GConnection(Async):
def __init__(self, *args, **kwargs):
"""
This class is a 'GEvent'-optimized subclass of libcouchbase
which utilizes the underlying IOPS structures and the gevent
event primitives to efficiently utilize couroutine switching.
"""
experimental.enabled_or_raise()
super(GConnection, self).__init__(IOPS(), *args, **kwargs)
def _do_ctor_connect(self):
if self.connected:
return
self._connect()
self._evconn = AsyncResult()
self._conncb = self._on_connected
self._evconn.get()
self._evconn = None
def _on_connected(self, err):
if err:
self._evconn.set_exception(err)
else:
self._evconn.set(None)
def _waitwrap(self, cbasync):
cur_thread = getcurrent()
cbasync.callback = cur_thread.switch
cbasync.errback = lambda r, x, y, z: cur_thread.throw(x, y, z)
return get_hub().switch()
def _meth_factory(meth, name):
def ret(self, *args, **kwargs):
return self._waitwrap(meth(self, *args, **kwargs))
return ret
def _http_request(self, **kwargs):
res = super(GConnection, self)._http_request(**kwargs)
if kwargs.get('chunked', False):
return res #views
e = Event()
res._callback = lambda x, y: e.set()
e.wait()
res._maybe_raise()
return res
def query(self, *args, **kwargs):
kwargs['itercls'] = GView
ret = super(GConnection, self).query(*args, **kwargs)
ret.start()
return ret
locals().update(Async._gen_memd_wrappers(_meth_factory))
def test_link_to_asyncresult(self):
p = gevent.spawn(lambda: 100)
event = AsyncResult()
p.link(event)
self.assertEqual(event.get(), 100)
for i in range(3):
event2 = AsyncResult()
p.link(event2)
self.assertEqual(event2.get(), 100)
def _start_platform(self):
"""
Starts the given platform waiting for it to transition to the
UNINITIALIZED state (note that the agent starts in the LAUNCHING state).
More in concrete the sequence of steps here are:
- prepares subscriber to receive the UNINITIALIZED state transition
- launches the platform process
- waits for the start of the process
- waits for the transition to the UNINITIALIZED state
"""
##############################################################
# prepare to receive the UNINITIALIZED state transition:
async_res = AsyncResult()
def consume_event(evt, *args, **kwargs):
log.debug("Got ResourceAgentStateEvent %s from origin %r", evt.state, evt.origin)
if evt.state == PlatformAgentState.UNINITIALIZED:
async_res.set(evt)
# start subscriber:
sub = EventSubscriber(event_type="ResourceAgentStateEvent",
origin=self.platform_device_id,
callback=consume_event)
sub.start()
log.info("registered event subscriber to wait for state=%r from origin %r",
PlatformAgentState.UNINITIALIZED, self.platform_device_id)
#self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
##############################################################
# now start the platform:
agent_instance_id = self.platform_agent_instance_id
log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id)
pid = self.imsclient.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
agent_instance_obj = self.imsclient.read_platform_agent_instance(agent_instance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
self.platform_device_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient(self.platform_device_id,
name=gate.process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
##############################################################
# wait for the UNINITIALIZED event:
async_res.get(timeout=self._receive_timeout)
def test_wait_noerrors(self):
x = gevent.spawn(lambda: 1)
y = gevent.spawn(lambda: 2)
z = gevent.spawn(lambda: 3)
gevent.joinall([x, y, z], raise_error=True)
self.assertEqual([x.value, y.value, z.value], [1, 2, 3])
e = AsyncResult()
x.link(e)
self.assertEqual(e.get(), 1)
x.unlink(e)
e = AsyncResult()
x.link(e)
self.assertEqual(e.get(), 1)
def test_set(self):
event1 = AsyncResult()
timer_exc = MyException('interrupted')
# Notice that this test is racy:
# After DELAY, we set the event. We also try to immediately
# raise the exception with a timer of 0 --- but that depends
# on cycling the loop. Hence the fairly large value for DELAY.
g = gevent.spawn_later(DELAY, event1.set, 'hello event1')
self._close_on_teardown(g.kill)
with gevent.Timeout.start_new(0, timer_exc):
with self.assertRaises(MyException) as exc:
event1.get()
self.assertIs(timer_exc, exc.exception)
class Future:
def __init__(self):
self.result = AsyncResult()
def set(self, value):
self.result.set(value)
def get(self):
return self.result.get()
def on_ready(self, func):
while self.result.ready() != False:
gevent.sleep(0)
func(self.result.get())
def request(self, service_name, data=""):
"""
这里将数据写到缓冲队列，然后让当前请求协程在event上等待
"""
if self._is_closed:
logging.error("request in a close cliet")
raise Exception("connection closed")
rid = self.rid
req = (service_name, data, rid)
req_data = cp_dumps(req)
req_data_len = len(req_data)
head = s_pack("i", req_data_len)
self._send_data.extend((head, req_data))
if not self._write_open:
self._write() # 立即尝试写一下，可以降低延迟
self._start_write()
event = AsyncResult()
self._events[rid] = event
try:
out = event.get(timeout=DEFAULT_REQUEST_TIMEOUT) # 在这个event上面等待，阻塞当前协程
if self._is_closed:
raise Exception("connection disconnected")
return out
finally:
del self._events[rid]
def call(self, protoname, msg):
session = self._get_session()
pack = proto.request(protoname, msg, session)
ev = AsyncResult()
self._sessions[session] = ev
self._send(pack)
return ev.get()
def rpc(request=None, conn=None, connections=None, timeout=None,
**kwargs):
request = request or request_class(**kwargs)
meta = self.meta
meta.Clear()
meta.service_method = service_method
meta.packet_type = packet_type
if connections is not None:
for conn in connections:
conn.send(conn.pack_meta(meta, request))
else:
conn = conn or self.conn or \
self.connection_pool.get_connection()
assert conn, conn
if require_response:
meta.transmission_id = conn.transmission_id
conn.transmission_id += 1
conn.send(conn.pack_meta(meta, request))
if hasattr(conn, 'release'):
conn.release()
if not require_response:
return
async_result = AsyncResult()
conn.transmissions[meta.transmission_id] = async_result
return async_result.get(timeout=timeout or self.timeout)
def multipublish(self, topic, messages, block=True, timeout=None,
raise_error=True):
"""Publish an iterable of messages to the given topic.
:param topic: the topic to publish to
:param messages: iterable of bytestrings to publish
:param block: wait for a connection to become available before
publishing the message. If block is `False` and no connections
are available, :class:`~gnsq.errors.NSQNoConnections` is raised
:param timeout: if timeout is a positive number, it blocks at most
``timeout`` seconds before raising
:class:`~gnsq.errors.NSQNoConnections`
:param raise_error: if ``True``, it blocks until a response is received
from the nsqd server, and any error response is raised. Otherwise
an :class:`~gevent.event.AsyncResult` is returned
"""
result = AsyncResult()
conn = self._get_connection(block=block, timeout=timeout)
try:
self._response_queues[conn].append(result)
conn.multipublish(topic, messages)
finally:
self._put_connection(conn)
if raise_error:
return result.get()
return result
def actor_exec(node, fn, *args, **kwargs):
class ExecActor(Actor):
def run(self):
ret.set(fn(*args, **kwargs))
ret = AsyncResult()
node.spawn(ExecActor)
return ret.get()
def request_many(self, reqs):
"""
salvo simultaneous query many service and get result
"""
local_quests = {}
idx = 0
while idx < len(reqs):
req = reqs[idx]
ar = AsyncResult()
qid = self.next_qid()
self.emit_query(qid, *req)
self._query_list[qid] = ar
local_quests[qid] = (idx, ar)
idx += 1
for qid, it in local_quests.items():
idx, ar = it
"""
async result will always get a response even if other query
reply soon
"""
msg = ar.get()
if msg:
if msg.status:
yield ProxyError(qid, msg.status, msg.params), idx
else:
yield msg.data, idx
def _sync_call(self, func, cb_arg, *args, **kwargs):
"""
Functionally similar to the generic blocking_cb but with error support that's Channel specific.
"""
ar = AsyncResult()
def cb(*args, **kwargs):
ret = list(args)
if len(kwargs): ret.append(kwargs)
ar.set(ret)
def eb(ch, code, text):
ar.set(ChannelError("_sync_call could not complete due to an error (%d): %s" % (code, text)))
kwargs[cb_arg] = cb
with self.push_closed_error_callback(eb):
func(*args, **kwargs)
ret_vals = ar.get(timeout=10)
if isinstance(ret_vals, ChannelError):
raise ret_vals
if len(ret_vals) == 0:
return None
elif len(ret_vals) == 1:
return ret_vals[0]
return tuple(ret_vals)
def sleepUntil(force,delta): from homevent.times import unixdelta,now if isinstance(delta,dt.datetime): delta = delta - now() if isinstance(delta,dt.timedelta): delta = unixdelta(delta) if delta < 0: # we're late delta = 0 # but let's hope not too late if "HOMEVENT_TEST" in os.environ: ev = AsyncResult() callLater(force,delta, ev.set,None) ev.get(block=True) else: gevent.sleep(delta)
def test_known_error(self):
# IonExceptions and TypeErrors get forwarded back intact
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
def proc_call():
raise NotFound("didn't find it")
def client_call(p=None, ar=None):
try:
ca = p._routing_call(proc_call, None)
ca.get(timeout=5)
except IonException as e:
ar.set(e)
ar = AsyncResult()
gl_call = spawn(client_call, p=p, ar=ar)
e = ar.get(timeout=5)
self.assertIsInstance(e, NotFound)
def test_watching_new_actor(defer):
class Watcher(Actor):
def pre_start(self):
a = self.spawn(Actor)
a.stop()
watchee.set(a)
self.watch(a)
def receive(self, message):
message_receieved.set((self.sender, message))
node = DummyNode()
defer(node.stop)
watchee, message_receieved = AsyncResult(), AsyncResult()
node.spawn(Watcher)
eq_(message_receieved.get(), (watchee.get(), ('terminated', watchee.get())))
def wait_event(self, event, timeout=None, raises=False):
"""
Blocks until an event and returns the results
:param event: event identifier
:param timeout: (optional)(default:None) seconds to wait
:type timeout: :class:`int`
:param raises: (optional)(default:False) On timeout if ``False`` return ``None``, else raise ``gevent.Timeout``
:type raises: :class:`bool`
:return: returns event arguments in tuple
:rtype: :class:`None`, or :class:`tuple`
:raises: ``gevent.Timeout``
Handling timeout
.. code:: python
args = ee.wait_event('my event', timeout=5)
if args is None:
print "Timeout!"
"""
result = AsyncResult()
self.once(event, result)
try:
return result.get(True, timeout)
except gevent.Timeout:
self.remove_listener(event, result)
if raises:
raise
else:
return None
def test_link_to_asyncresult_exception(self):
err = ExpectedError('test_link_to_asyncresult_exception')
p = gevent.spawn(lambda: getcurrent().throw(err))
event = AsyncResult()
p.link(event)
with self.assertRaises(ExpectedError) as exc:
event.get()
self.assertIs(exc.exception, err)
for _ in range(3):
event2 = AsyncResult()
p.link(event2)
with self.assertRaises(ExpectedError) as exc:
event2.get()
self.assertIs(exc.exception, err)
def test_unknown_error(self):
# Unhandled exceptions get handled and then converted to ContainerErrors
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
def proc_call():
raise self.ExpectedError("didn't find it")
def client_call(p=None, ar=None):
try:
ca = p._routing_call(proc_call, None)
ca.get(timeout=5)
except IonException as e:
ar.set(e)
ar = AsyncResult()
gl_call = spawn(client_call, p=p, ar=ar)
e = ar.get(timeout=5)
self.assertIsInstance(e, ContainerError)
self.assertEquals(len(p._errors), 1)
def _sync_call(self, func, cb_arg, *args, **kwargs):
"""
Functionally similar to the generic blocking_cb but with error support that's Channel specific.
"""
ar = AsyncResult()
def cb(*args, **kwargs):
ret = list(args)
if len(kwargs): ret.append(kwargs)
ar.set(ret)
eb = lambda ch, *args: ar.set(TransportError("_sync_call could not complete due to an error (%s)" % args))
kwargs[cb_arg] = cb
with self._push_close_cb(eb):
func(*args, **kwargs)
ret_vals = ar.get(timeout=10)
if isinstance(ret_vals, TransportError):
# mark this channel as poison, do not use again!
# don't test for type here, we don't want to have to import PyonSelectConnection
if hasattr(self._client.transport, 'connection') and hasattr(self._client.transport.connection, 'mark_bad_channel'):
self._client.transport.connection.mark_bad_channel(self._client.channel_number)
else:
log.warn("Could not mark channel # (%s) as bad, Pika could be corrupt", self._client.channel_number)
raise ret_vals
if len(ret_vals) == 0:
return None
elif len(ret_vals) == 1:
return ret_vals[0]
return tuple(ret_vals)
def fetch_hashchain(self):
log_st.debug('fetching hashchain')
blockhashes_chain = [self.blockhash] # youngest to oldest
blockhash = self.blockhash
assert blockhash not in self.chain
# get block hashes until we found a known one
max_blockhashes_per_request = self.initial_blockhashes_per_request
while blockhash not in self.chain:
# proto with highest_difficulty should be the proto we got the newblock from
blockhashes_batch = []
# try with protos
protocols = self.synchronizer.protocols
if not protocols:
log_st.warn('no protocols available')
return self.exit(success=False)
for proto in protocols:
if proto.is_stopped:
continue
# request
assert proto not in self.requests
deferred = AsyncResult()
self.requests[proto] = deferred
proto.send_getblockhashes(blockhash, max_blockhashes_per_request)
try:
blockhashes_batch = deferred.get(block=True,
timeout=self.blockhashes_request_timeout)
except gevent.Timeout:
log_st.warn('syncing timed out')
continue
finally:
# is also executed 'on the way out' when any other clause of the try statement
# is left via a break, continue or return statement.
del self.requests[proto]
if not blockhashes_batch:
log_st.warn('empty getblockhashes result')
continue
if not all(isinstance(bh, bytes) for bh in blockhashes_batch):
log_st.warn('got wrong data type', expected='bytes',
received=type(blockhashes_batch[0]))
continue
break
for blockhash in blockhashes_batch: # youngest to oldest
assert isinstance(blockhash, bytes)
if blockhash not in self.chain:
blockhashes_chain.append(blockhash)
else:
log_st.debug('found known blockhash', blockhash=blockhash.encode('hex'),
is_genesis=bool(blockhash == self.chain.genesis.hash))
break
max_blockhashes_per_request = self.max_blockhashes_per_request
self.fetch_blocks(blockhashes_chain)
def execution_result(executor):
result = AsyncResult()
deferred = executor()
assert isinstance(deferred, Deferred), 'Another middleware has converted the execution result ' \
'away from a Deferred.'
deferred.add_callbacks(result.set, lambda e: result.set_exception(e.value, (e.type, e.value, e.traceback)))
return result.get()
def test_set(self):
event1 = AsyncResult()
event2 = AsyncResult()
g = gevent.spawn_later(DELAY / 2.0, event1.set, 'hello event1')
t = gevent.Timeout.start_new(0, ValueError('interrupted'))
try:
try:
result = event1.get()
except ValueError as ex:
X = object()
result = gevent.with_timeout(DELAY,
event2.get,
timeout_value=X)
assert result is X, 'Nobody sent anything to event2 yet it received %r' % (
result, )
if PY3:
ex.__traceback__ = None
finally:
t.cancel()
g.kill()
def test_watching_running_remote_actor_that_stops_causes_termination_message(
defer):
class Watcher(Actor):
def pre_start(self):
self.watch(
self.root.node.lookup_str('localhost:20002/remote-watchee'))
def receive(self, msg):
received.set(msg)
received = AsyncResult()
node1, node2 = Node('localhost:20001',
enable_remoting=True), Node('localhost:20002',
enable_remoting=True)
defer(node1.stop, node2.stop)
remote_watchee = node2.spawn(Actor, name='remote-watchee')
node1.spawn(Watcher)
remote_watchee.stop()
eq_(received.get(), ('terminated', remote_watchee))
def send(self, destination, command, payload, allowed_replies):
result = AsyncResult()
def callback(reply):
result.set(reply)
def error_callback(reply):
result.set(reply)
defer = new_defer(
callback, error_callback, self.verify(allowed_replies),
allowed_replies,
new_message(destination, command,
self.getNextSequenceNumberAsPrefixPayload() + payload),
int(round(time.time() * 1000)) + 10000)
tasks.put_nowait(defer)
message = result.get() # blocking
# received ack from Agent
return message
def stop_tcp_server(self, data, reply_address):
DBG("NetoolCommandServer stop_tcp_server %r" % data)
try:
conn_greenlets[data['name']]['greenlet'].kill(exception=StopServer,
block=False)
result = AsyncResult()
conn_greenlets[data['name']]['result'] = result
re = result.get()
value = re['result']
if value != 'ok':
del (conn_greenlets[data['name']])
re = self.socket.sendto(json.dumps(re), reply_address)
DBG("sent %d" % re)
except:
DBG_TRACE()
self.socket.sendto(
json.dumps({
'result': 'error',
'msg': base64.b64encode(sys.exc_info()[1])
}), reply_address)
def receive():
"""Generator that yields a message at least every KEEP_ALIVE_DELAY seconds.
yields messages sent by `broadcast`.
"""
now = time.time()
end = now + MAX_DURATION
tmp = None
# Heroku doesn't notify when clients disconnect so we have to impose a
# maximum connection duration.
while now < end:
if not tmp:
tmp = AsyncResult()
BROADCAST_QUEUE.put(tmp)
try:
yield tmp.get(timeout=KEEP_ALIVE_DELAY)
tmp = None
except Timeout:
yield ''
now = time.time()
def waitForDeferred(d, result=None):
"""Block current greenlet for Deferred, waiting until result is not a Deferred or a failure is encountered"""
from twisted.internet import reactor
assert reactor.greenlet != getcurrent(), "can't invoke this in the reactor greenlet"
if result is None:
result = AsyncResult()
def cb(res):
if isinstance(res, defer.Deferred):
waitForDeferred(res, result)
else:
result.set(res)
def eb(res):
result.set_exception(res)
d.addCallbacks(cb, eb)
try:
return result.get()
except failure.Failure, ex:
ex.raiseException()
def _sync_call(self, func, cb_arg, *args, **kwargs):
"""
Functionally similar to the generic blocking_cb but with error support that's Channel specific.
"""
ar = AsyncResult()
def cb(*args, **kwargs):
ret = list(args)
if len(kwargs): ret.append(kwargs)
ar.set(ret)
eb = lambda ch, *args: ar.set(
TransportError("_sync_call could not complete due to an error (%s)"
% args))
kwargs[cb_arg] = cb
with self._push_close_cb(eb):
func(*args, **kwargs)
ret_vals = ar.get(timeout=10)
if isinstance(ret_vals, TransportError):
# mark this channel as poison, do not use again!
# don't test for type here, we don't want to have to import PyonSelectConnection
if hasattr(self._client.transport, 'connection') and hasattr(
self._client.transport.connection, 'mark_bad_channel'):
self._client.transport.connection.mark_bad_channel(
self._client.channel_number)
else:
log.warn(
"Could not mark channel # (%s) as bad, Pika could be corrupt",
self._client.channel_number)
raise ret_vals
if len(ret_vals) == 0:
return None
elif len(ret_vals) == 1:
return ret_vals[0]
return tuple(ret_vals)
def get(self):
def watcher(event):
if event.type in (EventType.CREATED, EventType.CHANGED):
_data, _stats = zk.get(job_status_path)
async_result.set((_data, _stats))
tid = self.get_argument("tid")
version = int(self.get_argument("version", -1))
job_status_path = os.path.join(ZK_JOB_STATUS_PATH, tid)
try:
raw_data, stats = zk.get(job_status_path)
except NoNodeError:
raise DoesNotExist("job status doesn't exists")
if version > stats.version:
raise ValidationError("job status version too large")
elif version == stats.version:
async_result = AsyncResult()
zk.exists(job_status_path, watch=watcher)
raw_data, stats = async_result.get()
try:
data = json.loads(raw_data)
# for compatible with 'undo' status set by GANGR
# regard this as common unfinished status like 'unknown','doing' and ignore it
if data['status'] == 'undo':
pass
elif Status[data['status']] in finished_statuses:
Deployment.set_status(tid, Status[data['status']])
except Exception as e:
Deployment.set_status(tid, Status.error)
raise TemporaryServerError("fetched status illegal: %s" % e)
ret = {
"version": stats.version,
"data": data,
}
self.write_json(ret)
def disconnect_accepted(self, data, reply_address):
DBG("NetoolCommandServer disconnect_accepted %r" % data)
try:
if not data['accepted_name'] in conn_greenlets: # already closed
self.socket.sendto(json.dumps({'result': 'ok'}), reply_address)
return
conn_greenlets[data['accepted_name']]['greenlet'].kill(
exception=CloseSocket, block=False)
result = AsyncResult()
conn_greenlets[data['accepted_name']]['result'] = result
value = result.get()
conn_greenlets[data['accepted_name']]['greenlet'].join()
del conn_greenlets[data['accepted_name']]
re = self.socket.sendto(json.dumps(value), reply_address)
DBG("sent %d" % re)
except:
DBG_TRACE()
self.socket.sendto(
json.dumps({
'result': 'error',
'msg': base64.b64encode(sys.exc_info()[1])
}), reply_address)
def multipublish(self,
topic,
messages,
block=True,
timeout=None,
raise_error=True):
"""Publish an iterable of messages to the given topic.
:param topic: the topic to publish to
:param messages: iterable of bytestrings to publish
:param block: wait for a connection to become available before
publishing the message. If block is `False` and no connections
are available, :class:`~gnsq.errors.NSQNoConnections` is raised
:param timeout: if timeout is a positive number, it blocks at most
``timeout`` seconds before raising
:class:`~gnsq.errors.NSQNoConnections`
:param raise_error: if ``True``, it blocks until a response is received
from the nsqd server, and any error response is raised. Otherwise
an :class:`~gevent.event.AsyncResult` is returned
"""
result = AsyncResult()
conn = self._get_connection(block=block, timeout=timeout)
try:
self._response_queues[conn].append(result)
conn.multipublish(topic, messages)
finally:
self._put_connection(conn)
if raise_error:
return result.get()
return result
def test_competing__routing_call(self):
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
sem = Semaphore()
# define a callable method that tries to grab a shared semaphore
def thecall(ar=None):
semres = sem.acquire(blocking=False)
if not semres:
raise StandardError(
"Could not get semaphore, routing_call/control flow is broken!"
)
# make this take a sec
time.sleep(1)
# make sure we release
sem.release()
# set the ar
ar.set(True)
# schedule some calls (in whatever order)
ar1 = AsyncResult()
ar2 = AsyncResult()
ar3 = AsyncResult()
p._routing_call(thecall, None, ar=ar3)
p._routing_call(thecall, None, ar=ar1)
p._routing_call(thecall, None, ar=ar2)
# wait on all the ARs to be set
ar1.get(timeout=5)
ar2.get(timeout=5)
ar3.get(timeout=5)
# just getting here without throwing an exception is the true test!
p._notify_stop()
p.stop()
def subprocess_stopped(result: AsyncResult) -> None:
if janitor._exit_in_progress:
# During __exit__ we expect processes to stop, since
# they are killed by the janitor.
return
with janitor._processes_lock:
# Processes are expected to quit while the nursery is
# active, remove them from the track list to clear memory
janitor._processes.remove(process)
# if the subprocess error'ed propagate the error.
try:
exit_code = result.get()
if exit_code != STATUS_CODE_FOR_SUCCESS:
log.error(
"Process died! Bailing out.",
args=process.args,
exit_code=exit_code,
)
exception = SystemExit(exit_code)
janitor._stop.set_exception(exception)
except Exception as exception:
janitor._stop.set_exception(exception)
def wait_for_events(events, timeout=None):
def _remove():
for event in events:
remove(event, on_event)
def on_event(*args, **kwargs):
_remove()
result.set([args, kwargs])
if timeout:
timeout = gevent.Timeout(
timeout,
RuntimeError('timeout while waiting for events: {}'.format(
', '.join(events))))
timeout.start()
try:
result = AsyncResult()
for event in events:
add(event, on_event)
return result.get()
finally:
if timeout:
timeout.cancel()
_remove()
class Qt4_DiffractometerMockup(HardwareObject):
"""
Descript. :
"""
MANUAL3CLICK_MODE = "Manual 3-click"
C3D_MODE = "Computer automatic"
MOVE_TO_BEAM_MODE = "Move to Beam"
def __init__(self, *args):
"""
Descript. :
"""
HardwareObject.__init__(self, *args)
qmo.CentredPosition.set_diffractometer_motor_names(\
"phi", "focus", "phiz", "phiy", "zoom",
"sampx", "sampy", "kappa", "kappa_phi")
self.phiMotor = None
self.phizMotor = None
self.phiyMotor = None
self.lightMotor = None
self.zoomMotor = None
self.sampleXMotor = None
self.sampleYMotor = None
self.camera = None
self.beam_position = None
self.x_calib = None
self.y_calib = None
self.pixels_per_mm_x = None
self.pixels_per_mm_y = None
self.image_width = None
self.image_height = None
self.current_sample_info = None
self.cancel_centring_methods = None
self.current_centring_procedure = None
self.current_centring_method = None
self.current_positions_dict = None
self.centring_methods = None
self.centring_status = None
self.centring_time = None
self.user_confirms_centring = None
self.user_clicked_event = None
self.phase_list = None
self.connect(self, 'equipmentReady', self.equipmentReady)
self.connect(self, 'equipmentNotReady', self.equipmentNotReady)
#IK - this will be sorted out
self.startCentringMethod = self.start_centring_method
self.cancelCentringMethod = self.cancel_centring_method
self.imageClicked = self.image_clicked
self.acceptCentring = self.accept_centring
self.rejectCentring = self.reject_centring
self.getCentringStatus = self.get_centring_status
self.takeSnapshots = self.take_snapshots
self.moveMotors = self.move_motors
def init(self):
"""
Descript. :
"""
self.x_calib = 0.000444
self.y_calib = 0.000446
self.pixels_per_mm_x = 1.0 / self.x_calib
self.pixels_per_mm_y = 1.0 / self.y_calib
self.beam_position = [200, 200]
self.centring_methods = {
Qt4_DiffractometerMockup.MANUAL3CLICK_MODE: self.start_3Click_centring,
Qt4_DiffractometerMockup.C3D_MODE: self.start_automatic_centring}
self.cancel_centring_methods = {}
self.current_positions_dict = {'phiy' : 0, 'phiz' : 0, 'sampx' : 0,
'sampy' : 0, 'zoom' : 0, 'phi' : 17.6,
'focus' : 0, 'kappa': 0, 'kappa_phi': 0,
'beam_x': 0, 'beam_y': 0}
self.current_state_dict = {}
self.centring_status = {"valid": False}
self.centring_time = 0
self.user_confirms_centring = True
self.user_clicked_event = AsyncResult()
self.image_width = 400
self.image_height = 400
self.equipmentReady()
self.user_clicked_event = AsyncResult()
self.phi_motor_hwobj = self.getObjectByRole('phi')
if self.phi_motor_hwobj is not None:
self.connect(self.phi_motor_hwobj, 'stateChanged', self.phi_motor_state_changed)
self.connect(self.phi_motor_hwobj, "positionChanged", self.phi_motor_moved)
self.reversing_rotation = self.getProperty("reversingRotation")
try:
self.grid_direction = eval(self.getProperty("gridDirection"))
except:
self.grid_direction = {"fast": (0, 1), "slow": (1, 0)}
try:
self.phase_list = eval(self.getProperty("phaseList"))
except:
self.phase_list = ['demo']
def getStatus(self):
"""
Descript. :
"""
return "ready"
def in_plate_mode(self):
return True
def is_reversing_rotation(self):
return True
def get_grid_direction(self):
"""
Descript. :
"""
return self.grid_direction
def manual_centring(self):
"""
Descript. :
"""
self.user_clicked_event = AsyncResult()
x, y = self.user_clicked_event.get()
last_centred_position[0] = x
last_centred_position[1] = y
random_num = random.random()
centred_pos_dir = {'phiy': random_num * 10, 'phiz': random_num,
'sampx': 0.0, 'sampy': 9.3, 'zoom': 8.53,
'phi': 311.1, 'focus': -0.42, 'kappa': 0.0009,
' kappa_phi': 311.0}
return centred_pos_dir
def set_sample_info(self, sample_info):
"""
Descript. :
"""
self.current_sample_info = sample_info
def emit_diffractometer_moved(self, *args):
"""
Descript. :
"""
self.emit("diffractometerMoved", ())
def isReady(self):
"""
Descript. :
"""
return True
def isValid(self):
"""
Descript. :
"""
return True
def equipmentReady(self):
"""
Descript. :
"""
self.emit('minidiffReady', ())
def equipmentNotReady(self):
"""
Descript. :
"""
self.emit('minidiffNotReady', ())
def phi_motor_moved(self, pos):
"""
Descript. :
"""
self.current_positions_dict["phi"] = pos
self.emit_diffractometer_moved()
self.emit("phiMotorMoved", pos)
#self.emit('stateChanged', (self.current_state_dict["phi"], ))
def phi_motor_state_changed(self, state):
"""
Descript. :
"""
self.current_state_dict["phi"] = state
self.emit('stateChanged', (state, ))
def invalidate_centring(self):
"""
Descript. :
"""
if self.current_centring_procedure is None and self.centring_status["valid"]:
self.centring_status = {"valid":False}
self.emitProgressMessage("")
self.emit('centringInvalid', ())
def get_centred_point_from_coord(self, x, y, return_by_names=None):
"""
Descript. :
"""
random_num = random.random()
centred_pos_dir = {'phiy': random_num * 10, 'phiz': random_num,
'sampx': 0.0, 'sampy': 9.3, 'zoom': 8.53,
'phi': 311.1, 'focus': -0.42, 'kappa': 0.0009,
'kappa_phi': 311.0}
return centred_pos_dir
def get_available_centring_methods(self):
"""
Descript. :
"""
return self.centring_methods.keys()
def get_calibration_data(self, offset):
"""
Descript. :
"""
#return (1.0 / self.x_calib, 1.0 / self.y_calib)
return (1.0 / self.x_calib, 1.0 / self.y_calib)
def get_pixels_per_mm(self):
"""
Descript. :
"""
return (self.pixels_per_mm_x, self.pixels_per_mm_y)
def refresh_omega_reference_position(self):
"""
Descript. :
"""
return
def get_omega_axis_position(self):
"""
Descript. :
"""
return self.current_positions_dict.get("phi")
def get_positions(self):
"""
Descript. :
"""
return self.current_positions_dict
def get_current_positions_dict(self):
"""
Descript. :
"""
return self.current_positions_dict
def beam_position_changed(self, value):
"""
Descript. :
"""
self.beam_position = value
def start_centring_method(self, method, sample_info = None):
"""
Descript. :
"""
if self.current_centring_method is not None:
logging.getLogger("HWR").error("already in centring method %s" %\
self.current_centring_method)
return
curr_time = time.strftime("%Y-%m-%d %H:%M:%S")
self.centring_status = {"valid": False, "startTime": curr_time}
self.emit_centring_started(method)
try:
fun = self.centring_methods[method]
except KeyError, diag:
logging.getLogger("HWR").error("unknown centring method (%s)" % \
str(diag))
self.emit_centring_failed()
else:
class _Socket(_original_Socket):
"""Green version of :class:`zmj.core.socket.Socket`
The following methods are overridden:
* send
* recv
To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or recieving
is deferred to the hub if a ``zmq.EAGAIN`` (retry) error is raised.
The `__state_changed` method is triggered when the zmq.FD for the socket is
marked as readable and triggers the necessary read and write events (which
are waited for in the recv and send methods).
Some double underscore prefixes are used to minimize pollution of
:class:`zmj.core.socket.Socket`'s namespace.
"""
def __init__(self, context, socket_type):
self.__setup_events()
def close(self):
# close the _state_event event, keeps the number of active file descriptors down
if not self._closed and getattr(self, '_state_event', None):
try:
self._state_event.stop()
except AttributeError as e:
# gevent<1.0 compat
self._state_event.cancel()
super(_Socket, self).close()
def __setup_events(self):
self.__readable = AsyncResult()
self.__writable = AsyncResult()
try:
self._state_event = get_hub().loop.io(self.getsockopt(FD),
1) # read state watcher
self._state_event.start(self.__state_changed)
except AttributeError:
# for gevent<1.0 compatibility
from gevent.core import read_event
self._state_event = read_event(self.getsockopt(FD),
self.__state_changed,
persist=True)
def __state_changed(self, event=None, _evtype=None):
try:
if self.closed:
# if the socket has entered a close state resume any waiting greenlets
self.__writable.set()
self.__readable.set()
return
events = self.getsockopt(zmq.EVENTS)
except ZMQError as exc:
self.__writable.set_exception(exc)
self.__readable.set_exception(exc)
else:
if events & zmq.POLLOUT:
self.__writable.set()
if events & zmq.POLLIN:
self.__readable.set()
def _wait_write(self):
self.__writable = AsyncResult()
self.__writable.get()
def _wait_read(self):
self.__readable = AsyncResult()
self.__readable.get()
def send(self, data, flags=0, copy=True, track=False):
# if we're given the NOBLOCK flag act as normal and let the EAGAIN get raised
if flags & zmq.NOBLOCK:
return super(_Socket, self).send(data, flags, copy, track)
# ensure the zmq.NOBLOCK flag is part of flags
flags |= zmq.NOBLOCK
while True: # Attempt to complete this operation indefinitely, blocking the current greenlet
try:
# attempt the actual call
return super(_Socket, self).send(data, flags, copy, track)
except zmq.ZMQError as e:
# if the raised ZMQError is not EAGAIN, reraise
if e.errno != zmq.EAGAIN:
raise
# defer to the event loop until we're notified the socket is writable
self._wait_write()
def recv(self, flags=0, copy=True, track=False):
if flags & zmq.NOBLOCK:
return super(_Socket, self).recv(flags, copy, track)
flags |= zmq.NOBLOCK
while True:
try:
return super(_Socket, self).recv(flags, copy, track)
except zmq.ZMQError as e:
if e.errno != zmq.EAGAIN:
raise
self._wait_read()
def run_watcher(self):
res = AsyncResult()
self.watch_q.put(res)
return res.get()
class _Socket(_original_Socket):
"""Green version of :class:`zmq.core.socket.Socket`
The following methods are overridden:
* send
* recv
To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or recieving
is deferred to the hub if a ``zmq.EAGAIN`` (retry) error is raised.
The `__state_changed` method is triggered when the zmq.FD for the socket is
marked as readable and triggers the necessary read and write events (which
are waited for in the recv and send methods).
Some double underscore prefixes are used to minimize pollution of
:class:`zmq.core.socket.Socket`'s namespace.
"""
def __init__(self, context, socket_type):
self.__in_send_multipart = False
self.__in_recv_multipart = False
self.__setup_events()
def __del__(self):
self.close()
def close(self, linger=None):
super(_Socket, self).close(linger)
self.__cleanup_events()
def __cleanup_events(self):
# close the _state_event event, keeps the number of active file descriptors down
if getattr(self, '_state_event', None):
_stop(self._state_event)
self._state_event = None
# if the socket has entered a close state resume any waiting greenlets
self.__writable.set()
self.__readable.set()
def __setup_events(self):
self.__readable = AsyncResult()
self.__writable = AsyncResult()
self.__readable.set()
self.__writable.set()
try:
self._state_event = get_hub().loop.io(self.getsockopt(zmq.FD),
1) # read state watcher
self._state_event.start(self.__state_changed)
except AttributeError:
# for gevent<1.0 compatibility
from gevent.core import read_event
self._state_event = read_event(self.getsockopt(zmq.FD),
self.__state_changed,
persist=True)
def __state_changed(self, event=None, _evtype=None):
if self.closed:
self.__cleanup_events()
return
try:
# avoid triggering __state_changed from inside __state_changed
events = super(_Socket, self).getsockopt(zmq.EVENTS)
except zmq.ZMQError as exc:
self.__writable.set_exception(exc)
self.__readable.set_exception(exc)
else:
if events & zmq.POLLOUT:
self.__writable.set()
if events & zmq.POLLIN:
self.__readable.set()
def _wait_write(self):
assert self.__writable.ready(
), "Only one greenlet can be waiting on this event"
self.__writable = AsyncResult()
# timeout is because libzmq cannot be trusted to properly signal a new send event:
# this is effectively a maximum poll interval of 1s
tic = time.time()
timeout = gevent.Timeout(seconds=1)
try:
timeout.start()
self.__writable.get(block=True)
except gevent.Timeout as t:
if t is not timeout:
raise
toc = time.time()
# gevent bug: get can raise timeout even on clean return
# don't display zmq bug warning for gevent bug (this is getting ridiculous)
if toc - tic > 0.9 and self.getsockopt(zmq.EVENTS) & zmq.POLLOUT:
print("BUG: gevent missed a libzmq send event on %i!" %
self.FD,
file=sys.stderr)
finally:
timeout.cancel()
self.__writable.set()
def _wait_read(self):
assert self.__readable.ready(
), "Only one greenlet can be waiting on this event"
self.__readable = AsyncResult()
# timeout is because libzmq cannot always be trusted to play nice with libevent.
# I can only confirm that this actually happens for send, but lets be symmetrical
# with our dirty hacks.
# this is effectively a maximum poll interval of 1s
tic = time.time()
timeout = gevent.Timeout(seconds=1)
try:
timeout.start()
self.__readable.get(block=True)
except gevent.Timeout as t:
if t is not timeout:
raise
toc = time.time()
# gevent bug: get can raise timeout even on clean return
# don't display zmq bug warning for gevent bug (this is getting ridiculous)
if toc - tic > 0.9 and self.getsockopt(zmq.EVENTS) & zmq.POLLIN:
print("BUG: gevent missed a libzmq recv event on %i!" %
self.FD,
file=sys.stderr)
finally:
timeout.cancel()
self.__readable.set()
def send(self, data, flags=0, copy=True, track=False):
"""send, which will only block current greenlet
state_changed always fires exactly once (success or fail) at the
end of this method.
"""
# if we're given the NOBLOCK flag act as normal and let the EAGAIN get raised
if flags & zmq.NOBLOCK:
try:
msg = super(_Socket, self).send(data, flags, copy, track)
finally:
if not self.__in_send_multipart:
self.__state_changed()
return msg
# ensure the zmq.NOBLOCK flag is part of flags
flags |= zmq.NOBLOCK
while True: # Attempt to complete this operation indefinitely, blocking the current greenlet
try:
# attempt the actual call
msg = super(_Socket, self).send(data, flags, copy, track)
except zmq.ZMQError as e:
# if the raised ZMQError is not EAGAIN, reraise
if e.errno != zmq.EAGAIN:
if not self.__in_send_multipart:
self.__state_changed()
raise
else:
if not self.__in_send_multipart:
self.__state_changed()
return msg
# defer to the event loop until we're notified the socket is writable
self._wait_write()
def recv(self, flags=0, copy=True, track=False):
"""recv, which will only block current greenlet
state_changed always fires exactly once (success or fail) at the
end of this method.
"""
if flags & zmq.NOBLOCK:
try:
msg = super(_Socket, self).recv(flags, copy, track)
finally:
if not self.__in_recv_multipart:
self.__state_changed()
return msg
flags |= zmq.NOBLOCK
while True:
try:
msg = super(_Socket, self).recv(flags, copy, track)
except zmq.ZMQError as e:
if e.errno != zmq.EAGAIN:
if not self.__in_recv_multipart:
self.__state_changed()
raise
else:
if not self.__in_recv_multipart:
self.__state_changed()
return msg
self._wait_read()
def send_multipart(self, *args, **kwargs):
"""wrap send_multipart to prevent state_changed on each partial send"""
self.__in_send_multipart = True
try:
msg = super(_Socket, self).send_multipart(*args, **kwargs)
finally:
self.__in_send_multipart = False
self.__state_changed()
return msg
def recv_multipart(self, *args, **kwargs):
"""wrap recv_multipart to prevent state_changed on each partial recv"""
self.__in_recv_multipart = True
try:
msg = super(_Socket, self).recv_multipart(*args, **kwargs)
finally:
self.__in_recv_multipart = False
self.__state_changed()
return msg
def getsockopt(self, opt):
"""trigger state_changed on getsockopt(EVENTS)"""
optval = super(_Socket, self).getsockopt(opt)
if opt == zmq.EVENTS:
self.__state_changed()
return optval
class AceClient(object):
def __init__(self, acehostslist, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = float(result_timeout)
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current EVENT
self._event = None
# Current STATE
self._state = None
# Current AUTH
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
# Seekback seconds.
self._seekback = None
# Did we get START command again? For seekback.
self._started_again = Event()
self._idleSince = time.time()
self._streamReaderConnection = None
self._streamReaderState = Event()
self._streamReaderQueue = gevent.queue.Queue(
maxsize=1024) # Ring buffer with max number of chunks in queue
self._engine_version_code = None
# Logger
logger = logging.getLogger('AceClient')
# Try to connect AceStream engine
for AceEngine in acehostslist:
try:
self._socket = Telnet(AceEngine[0], AceEngine[1],
connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[
0], AceEngine[1], AceEngine[2]
logger.debug('Successfully connected to AceStream on %s:%d' %
(AceEngine[0], AceEngine[1]))
break
except:
logger.debug('The are no alive AceStream on %s:%d' %
(AceEngine[0], AceEngine[1]))
pass
# Spawning recvData greenlet
if self._socket: gevent.spawn(self._recvData)
else:
logger.error('The are no alive AceStream Engines found')
return
def destroy(self):
'''
AceClient Destructor
'''
logger = logging.getLogger('AceClient_destroy') # Logger
if self._shuttingDown.ready():
return # Already in the middle of destroying
self._result.set()
# Trying to disconnect
try:
logger.debug('Destroying AceStream client.....')
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
pass # Ignore exceptions on destroy
finally:
self._shuttingDown.set()
def reset(self):
self._idleSince = time.time()
self._started_again.clear()
self._streamReaderState.clear()
self._result.set()
def _write(self, message):
try:
logger = logging.getLogger('AceClient_write')
logger.debug('>>> %s' % message)
self._socket.write('%s\r\n' % message)
except EOFError as e:
raise AceException('Write error! %s' % repr(e))
def aceInit(self,
gender=AceConst.SEX_MALE,
age=AceConst.AGE_25_34,
product_key=AceConfig.acekey):
self._product_key = product_key
self._gender = gender
self._age = age
self._seekback = AceConfig.videoseekback
self._started_again.clear()
logger = logging.getLogger('AceClient_aceInit')
self._result = AsyncResult()
self._write(AceMessage.request.HELLO) # Sending HELLOBG
try:
params = self._getResult(timeout=self._resulttimeout)
except:
errmsg = 'HELLOTS not resived from engine!'
raise AceException(errmsg)
return
self._engine_version_code = int(params.get('version_code', 0))
self._result = AsyncResult()
self._write(
AceMessage.request.READY(params.get('key', ''), self._product_key))
if not self._getResult(timeout=self._resulttimeout
): # Get NOTREADY instead AUTH user_auth_level
errmsg = 'NOTREADY recived from AceEngine! Wrong acekey?'
raise AceException(errmsg)
return
if self._engine_version_code >= 3003600: # Display download_stopped massage
params_dict = {'use_stop_notifications': '1'}
self._write(AceMessage.request.SETOPTIONS(params_dict))
def _getResult(self, timeout=10.0):
logger = logging.getLogger('AceClient_getResult') # Logger
try:
return self._result.get(timeout=timeout)
except gevent.Timeout:
errmsg = 'Engine response time exceeded. getResult timeout from %s:%s' % (
AceConfig.acehost, AceConfig.aceAPIport)
raise AceException(errmsg)
def START(self, datatype, value, stream_type):
'''
Start video method
Returns the url provided by AceEngine
'''
if stream_type == 'hls' and self._engine_version_code >= 3010500:
params_dict = {
'output_format': stream_type,
'transcode_audio': AceConfig.transcode_audio,
'transcode_mp3': AceConfig.transcode_mp3,
'transcode_ac3': AceConfig.transcode_ac3,
'preferred_audio_language': AceConfig.preferred_audio_language
}
else:
params_dict = {'output_format': 'http'}
self._result = AsyncResult()
self._write(
AceMessage.request.START(
datatype.upper(), value, ' '.join(
['{}={}'.format(k, v) for k, v in params_dict.items()])))
return self._getResult(timeout=float(
AceConfig.videotimeout)) # Get url for play from AceEngine
def STOP(self):
'''
Stop video method
'''
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult(timeout=self._resulttimeout
) # Get STATE 0(IDLE) after sendig STOP to AceEngine
def LOADASYNC(self, datatype, params):
self._result = AsyncResult()
self._write(
AceMessage.request.LOADASYNC(
datatype.upper(),
random.randint(1, AceConfig.maxconns * 10000), params))
return self._getResult(timeout=self._resulttimeout
) # Get _contentinfo json from AceEngine
def GETCONTENTINFO(self, datatype, value):
params_dict = {
datatype: value,
'developer_id': '0',
'affiliate_id': '0',
'zone_id': '0'
}
return self.LOADASYNC(datatype, params_dict)
def GETCID(self, datatype, url):
contentinfo = self.GETCONTENTINFO(datatype, url)
if contentinfo['status'] in (1, 2):
params_dict = {
'checksum': contentinfo['checksum'],
'infohash': contentinfo['infohash'],
'developer_id': '0',
'affiliate_id': '0',
'zone_id': '0'
}
self._result = AsyncResult()
self._write(AceMessage.request.GETCID(params_dict))
cid = self._result.get(timeout=5.0)
else:
cid = None
errmsg = 'LOADASYNC returned error with message: %s' % contentinfo[
'message']
raise AceException(errmsg)
return '' if cid is None or cid == '' else cid[2:]
def GETINFOHASH(self, datatype, url, idx=0):
contentinfo = self.GETCONTENTINFO(datatype, url)
if contentinfo['status'] in (1, 2):
return contentinfo['infohash'], [
x[0] for x in contentinfo['files'] if x[1] == int(idx)
][0]
elif contentinfo['status'] == 0:
errmsg = 'LOADASYNC returned status 0: The transport file does not contain audio/video files'
raise AceException(errmsg)
else:
errmsg = 'LOADASYNC returned error with message: %s' % contentinfo[
'message']
raise AceException(errmsg)
return None, None
def startStreamReader(self, url, cid, counter, req_headers=None):
logger = logging.getLogger('StreamReader')
logger.debug('Open video stream: %s' % url)
transcoder = None
logger.debug('Get headers from client: %s' % req_headers)
try:
if url.endswith('.m3u8'):
logger.warning(
'HLS stream detected. Ffmpeg transcoding started')
popen_params = {
'bufsize': requests.models.CONTENT_CHUNK_SIZE,
'stdout': gevent.subprocess.PIPE,
'stderr': None,
'shell': False
}
if AceConfig.osplatform == 'Windows':
ffmpeg_cmd = 'ffmpeg.exe '
CREATE_NO_WINDOW = 0x08000000
CREATE_NEW_PROCESS_GROUP = 0x00000200
DETACHED_PROCESS = 0x00000008
popen_params.update(creationflags=CREATE_NO_WINDOW
| DETACHED_PROCESS
| CREATE_NEW_PROCESS_GROUP)
else:
ffmpeg_cmd = 'ffmpeg '
ffmpeg_cmd += '-hwaccel auto -hide_banner -loglevel fatal -re -i %s -c copy -f mpegts -' % url
transcoder = gevent.subprocess.Popen(ffmpeg_cmd.split(),
**popen_params)
out = transcoder.stdout
else:
self._streamReaderConnection = requests.get(
url,
headers=req_headers,
stream=True,
timeout=(5, AceConfig.videotimeout))
self._streamReaderConnection.raise_for_status(
) # raise an exception for error codes (4xx or 5xx)
out = self._streamReaderConnection.raw
self._streamReaderState.set()
self._write(AceMessage.request.EVENT('play'))
while 1:
gevent.sleep()
clients = counter.getClients(cid)
if clients:
try:
chunk = out.read(requests.models.CONTENT_CHUNK_SIZE)
try:
self._streamReaderQueue.put_nowait(chunk)
except gevent.queue.Full:
self._streamReaderQueue.get_nowait()
self._streamReaderQueue.put_nowait(chunk)
except requests.packages.urllib3.exceptions.ReadTimeoutError:
logger.warning(
'No data received from AceEngine for %ssec - broadcast stoped'
% AceConfig.videotimeout)
break
except:
break
else:
for c in clients:
try:
c.queue.put(chunk, timeout=5)
except gevent.queue.Full:
if len(clients) > 1:
logger.warning(
'Client %s does not read data from buffer until 5sec - disconnect it'
% c.handler.clientip)
c.destroy()
except gevent.GreenletExit:
pass
else:
logger.debug('All clients disconnected - broadcast stoped')
break
except requests.exceptions.HTTPError as err:
logger.error(
'An http error occurred while connecting to aceengine: %s' %
repr(err))
except requests.exceptions.RequestException:
logger.error(
'There was an ambiguous exception that occurred while handling request'
)
except Exception as err:
logger.error('Unexpected error in streamreader %s' % repr(err))
finally:
self.closeStreamReader()
if transcoder is not None:
try:
transcoder.kill()
logger.warning('Ffmpeg transcoding stoped')
except:
pass
counter.deleteAll(cid)
def closeStreamReader(self):
logger = logging.getLogger('StreamReader')
self._streamReaderState.clear()
if self._streamReaderConnection:
logger.debug('Close video stream: %s' %
self._streamReaderConnection.url)
self._streamReaderConnection.close()
self._streamReaderQueue.queue.clear()
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while 1:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until('\r\n').strip()
logger.debug('<<< %s' %
requests.compat.unquote(self._recvbuffer))
except:
# If something happened during read, abandon reader.
logger.error('Exception at socket read. AceClient destroyed')
if not self._shuttingDown.ready(): self._shuttingDown.set()
return
else: # Parsing everything only if the string is not empty
# HELLOTS
if self._recvbuffer.startswith('HELLOTS'):
# version=engine_version version_code=version_code key=request_key http_port=http_port
self._result.set({
k: v
for k, v in (x.split('=')
for x in self._recvbuffer.split()
if '=' in x)
})
# NOTREADY
elif self._recvbuffer.startswith('NOTREADY'):
self._result.set(False)
# AUTH
elif self._recvbuffer.startswith('AUTH'):
self._result.set(
self._recvbuffer.split()[1]) # user_auth_level
# START
elif self._recvbuffer.startswith('START'):
# url [ad=1 [interruptable=1]] [stream=1] [pos=position]
params = {
k: v
for k, v in (x.split('=')
for x in self._recvbuffer.split()
if '=' in x)
}
if not self._seekback or self._started_again.ready(
) or params.get('stream', '') is not '1':
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seekback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
self._result.set(
self._recvbuffer.split()[1]) # url for play
# LOADRESP
elif self._recvbuffer.startswith('LOADRESP'):
self._result.set(
requests.compat.json.loads(
requests.compat.unquote(' '.join(
self._recvbuffer.split()[2:]))))
# STATE
elif self._recvbuffer.startswith('STATE'):
self._state = self._recvbuffer.split()[1] # state_id
if self._state == '0': # 0(IDLE)
self._result.set(
self._write(AceMessage.request.EVENT('stop')))
elif self._state == '1':
pass # 1 (PREBUFFERING)
elif self._state == '2':
pass # 2 (DOWNLOADING)
elif self._state == '3':
pass # 3 (BUFFERING)
elif self._state == '4':
pass # 4 (COMPLETED)
elif self._state == '5':
pass # 5 (CHECKING)
elif self._state == '6':
pass # 6 (ERROR)
# STATUS
elif self._recvbuffer.startswith('STATUS'):
pass
# CID
elif self._recvbuffer.startswith('##'):
self._result.set(self._recvbuffer)
# INFO
elif self._recvbuffer.startswith('INFO'):
pass
# EVENT
elif self._recvbuffer.startswith('EVENT'):
self._event = self._recvbuffer.split()
if 'livepos' in self._event:
if self._seekback and not self._started_again.ready(
): # if seekback
params = {
k: v
for k, v in (x.split('=') for x in self._event
if '=' in x)
}
self._write(
AceMessage.request.LIVESEEK(
int(params['last']) - self._seekback))
self._started_again.set()
elif 'getuserdata' in self._event:
self._write(
AceMessage.request.USERDATA(
self._gender, self._age))
elif 'cansave' in self._event:
pass
elif 'showurl' in self._event:
pass
elif 'download_stopped' in self._event:
pass
# PAUSE
elif self._recvbuffer.startswith('PAUSE'):
self._write(AceMessage.request.EVENT('pause'))
# RESUME
elif self._recvbuffer.startswith('RESUME'):
self._write(AceMessage.request.EVENT('play'))
# STOP
elif self._recvbuffer.startswith('STOP'):
pass
# SHUTDOWN
elif self._recvbuffer.startswith('SHUTDOWN'):
self._socket.close()
logger.debug('AceClient destroyed')
return
class Step(ExecutionElement):
_templatable = True
def __init__(self,
app,
action,
name='',
device='',
inputs=None,
triggers=None,
next_steps=None,
position=None,
widgets=None,
risk=0,
uid=None,
templated=False,
raw_representation=None):
"""Initializes a new Step object. A Workflow has many steps that it executes.
Args:
app (str): The name of the app associated with the Step
action (str): The name of the action associated with a Step
name (str, optional): The name of the Step object. Defaults to an empty string.
device (str, optional): The name of the device associated with the app associated with the Step. Defaults
to an empty string.
inputs (dict, optional): A dictionary of Argument objects that are input to the step execution. Defaults
to None.
triggers (list[Flag], optional): A list of Flag objects for the Step. If a Step should wait for data
before continuing, then include these Trigger objects in the Step init. Defaults to None.
next_steps (list[NextStep], optional): A list of NextStep objects for the Step object. Defaults to None.
position (dict, optional): A dictionary with the x and y coordinates of the Step object. This is used
for UI display purposes. Defaults to None.
widgets (list[tuple(str, str)], optional): A list of widget tuples, which holds the app and the
corresponding widget. Defaults to None.
risk (int, optional): The risk associated with the Step. Defaults to 0.
uid (str, optional): A universally unique identifier for this object.
Created from uuid.uuid4().hex in Python
templated (bool, optional): Whether or not the Step is templated. Used for Jinja templating.
raw_representation (dict, optional): JSON representation of this object. Used for Jinja templating.
"""
ExecutionElement.__init__(self, uid)
self.triggers = triggers if triggers is not None else []
self._incoming_data = AsyncResult()
self.name = name
self.app = app
self.action = action
self._run, self._input_api = get_app_action_api(self.app, self.action)
get_app_action(self.app, self._run)
if isinstance(inputs, list):
inputs = {arg['name']: arg['value'] for arg in inputs}
elif isinstance(inputs, dict):
inputs = inputs
else:
inputs = {}
self.templated = templated
if not self.templated:
self.inputs = validate_app_action_parameters(
self._input_api, inputs, self.app, self.action)
else:
self.inputs = inputs
self.device = device if (device is not None
and device != 'None') else ''
self.risk = risk
self.next_steps = next_steps if next_steps is not None else []
self.position = position if position is not None else {}
self.widgets = [
widget if isinstance(widget, Widget) else Widget(**widget)
for widget in widgets
] if widgets is not None else []
self._output = None
self._next_up = None
self._raw_representation = raw_representation if raw_representation is not None else {}
self._execution_uid = 'default'
def get_output(self):
"""Gets the output of a Step (the result)
Returns:
The result of the Step
"""
return self._output
def get_next_up(self):
"""Gets the next step to be executed
Returns:
The next step to be executed
"""
return self._next_up
def set_next_up(self, next_up):
"""Sets the next step to be executed
Returns:
The next step to be executed
"""
self._next_up = next_up
def get_execution_uid(self):
"""Gets the execution UID of the Step
Returns:
The execution UID
"""
return self._execution_uid
def send_data_to_trigger(self, data):
"""Sends data to the Step if it has triggers associated with it, and is currently awaiting data
Args:
data (dict): The data to send to the triggers. This dict has two keys: 'data_in' which is the data
to be sent to the triggers, and 'inputs', which is an optional parameter to change the inputs to the
current Step
"""
self._incoming_data.set(data)
def _update_json(self, updated_json):
self.action = updated_json['action']
self.app = updated_json['app']
self.device = updated_json['device'] if 'device' in updated_json else ''
self.risk = updated_json['risk'] if 'risk' in updated_json else 0
inputs = {arg['name']: arg['value']
for arg in updated_json['inputs']
} if 'inputs' in updated_json else {}
if inputs is not None:
if not self.templated:
self.inputs = validate_app_action_parameters(
self._input_api, inputs, self.app, self.action)
else:
self.inputs = inputs
else:
self.inputs = validate_app_action_parameters(
self._input_api, {}, self.app, self.action)
self.next_steps = [
NextStep.create(cond_json)
for cond_json in updated_json['next_steps']
]
@contextdecorator.context
def render_step(self, **kwargs):
"""Uses JINJA templating to render a Step object.
Args:
kwargs (dict[str]): Arguments to use in the JINJA templating.
"""
if self.templated:
from jinja2 import Environment
env = Environment().from_string(
json.dumps(self._raw_representation)).render(
core.config.config.JINJA_GLOBALS, **kwargs)
self._update_json(updated_json=json.loads(env))
def set_input(self, new_input):
"""Updates the input for a Step object.
Args:
new_input (dict): The new inputs for the Step object.
"""
self.inputs = validate_app_action_parameters(self._input_api,
new_input, self.app,
self.action)
def execute(self, instance, accumulator):
"""Executes a Step by calling the associated app function.
Args:
instance (App): The instance of an App object to be used to execute the associated function.
accumulator (dict): Dict containing the results of the previous steps
Returns:
The result of the executed function.
"""
self._execution_uid = uuid.uuid4().hex
data_sent.send(self, callback_name="Step Started", object_type="Step")
if self.triggers:
data_sent.send(self,
callback_name="Trigger Step Awaiting Data",
object_type="Step")
logger.debug('Trigger Step {} is awaiting data'.format(self.name))
while True:
try:
data = self._incoming_data.get(timeout=1)
self._incoming_data = AsyncResult()
except gevent.Timeout:
gevent.sleep(0.1)
continue
data_in = data['data_in']
inputs = data['inputs'] if 'inputs' in data else {}
if all(
flag.execute(data_in=data_in, accumulator=accumulator)
for flag in self.triggers):
data_sent.send(self,
callback_name="Trigger Step Taken",
object_type="Step")
logger.debug(
'Trigger is valid for input {0}'.format(data_in))
accumulator[self.name] = data_in
if inputs:
self.inputs.update(inputs)
break
else:
logger.debug(
'Trigger is not valid for input {0}'.format(data_in))
data_sent.send(self,
callback_name="Trigger Step Not Taken",
object_type="Step")
gevent.sleep(0.1)
try:
args = dereference_step_routing(self.inputs, accumulator,
'In step {0}'.format(self.name))
args = validate_app_action_parameters(self._input_api, args,
self.app, self.action)
action = get_app_action(self.app, self._run)
if is_app_action_bound(self.app, self._run):
result = action(instance, **args)
else:
result = action(**args)
data_sent.send(self,
callback_name="Function Execution Success",
object_type="Step",
data=json.dumps({"result": result.as_json()}))
except InvalidInput as e:
formatted_error = format_exception_message(e)
logger.error('Error calling step {0}. Error: {1}'.format(
self.name, formatted_error))
data_sent.send(self,
callback_name="Step Input Invalid",
object_type="Step")
self._output = ActionResult('error: {0}'.format(formatted_error),
'InvalidInput')
raise
except Exception as e:
formatted_error = format_exception_message(e)
logger.error('Error calling step {0}. Error: {1}'.format(
self.name, formatted_error))
self._output = ActionResult('error: {0}'.format(formatted_error),
'UnhandledException')
raise
else:
self._output = result
for widget in self.widgets:
get_widget_signal(widget.app, widget.name).send(
self, data=json.dumps({"result": result.as_json()}))
logger.debug('Step {0}-{1} (uid {2}) executed successfully'.format(
self.app, self.action, self.uid))
return result
def get_next_step(self, accumulator):
"""Gets the NextStep object to be executed after the current Step.
Args:
accumulator (dict): A record of teh previously-executed steps. Of form {step_name: result}
Returns:
The NextStep object to be executed.
"""
for next_step in self.next_steps:
next_step = next_step.execute(self._output, accumulator)
if next_step is not None:
self._next_up = next_step
data_sent.send(self,
callback_name="Conditionals Executed",
object_type="Step")
return next_step
class DiffractometerMockup(Equipment):
"""
Descript. :
"""
MANUAL3CLICK_MODE = "Manual 3-click"
C3D_MODE = "Computer automatic"
MOVE_TO_BEAM_MODE = "Move to Beam"
MINIKAPPA = "MiniKappa"
PLATE = "Plate"
SC = "SC"
def __init__(self, *args):
"""
Descript. :
"""
Equipment.__init__(self, *args)
queue_model_objects.CentredPosition.set_diffractometer_motor_names(\
"phi", "focus", "phiz", "phiy", "zoom",
"sampx", "sampy", "kappa", "kappa_phi")
self.phiMotor = None
self.phizMotor = None
self.phiyMotor = None
self.lightMotor = None
self.zoomMotor = None
self.sampleXMotor = None
self.sampleYMotor = None
self.camera_hwobj = None
self.beam_info_hwobj = None
self.beam_position = None
self.x_calib = None
self.y_calib = None
self.pixels_per_mm_x = None
self.pixels_per_mm_y = None
self.image_width = None
self.image_height = None
self.current_sample_info = None
self.cancel_centring_methods = None
self.current_centring_procedure = None
self.current_centring_method = None
self.current_positions_dict = None
self.current_phase = None
self.centring_methods = None
self.centring_status = None
self.centring_time = None
self.user_confirms_centring = None
self.user_clicked_event = None
self.phase_list = None
self.head_type = None
self._drawing = None
self.connect(self, 'equipmentReady', self.equipmentReady)
self.connect(self, 'equipmentNotReady', self.equipmentNotReady)
self.startCentringMethod = self.start_centring_method
self.cancelCentringMethod = self.cancel_centring_method
self.imageClicked = self.image_clicked
self.acceptCentring = self.accept_centring
self.rejectCentring = self.reject_centring
self.getCentringStatus = self.get_centring_status
self.takeSnapshots = self.take_snapshots
self.moveMotors = self.move_motors
def init(self):
"""
Descript. :
"""
self.x_calib = 0.000444
self.y_calib = 0.000446
self.pixels_per_mm_x = 1.0 / self.x_calib
self.pixels_per_mm_y = 1.0 / self.y_calib
self.beam_position = [200, 200]
self.centring_methods = {
DiffractometerMockup.MANUAL3CLICK_MODE: self.start_3Click_centring,
DiffractometerMockup.C3D_MODE: self.start_automatic_centring}
self.cancel_centring_methods = {}
self.current_positions_dict = {'phiy' : 0, 'phiz' : 0, 'sampx' : 0,
'sampy' : 0, 'zoom' : 0, 'phi' : 17.6,
'focus' : 0, 'kappa': 0, 'kappa_phi': 0,
'beam_x': 0, 'beam_y': 0}
self.centring_status = {"valid": False}
self.centring_time = 0
self.user_confirms_centring = True
self.fast_shutter_is_open = False
self.user_clicked_event = AsyncResult()
self.image_width = 400
self.image_height = 400
self.equipmentReady()
self.user_clicked_event = AsyncResult()
self.phiMotor = self.getDeviceByRole('phi')
self.sampleXMotor = self.getDeviceByRole('sampx')
self.sampleYMotor = self.getDeviceByRole('sampy')
self.camera_hwobj = self.getDeviceByRole('camera')
if self.phiMotor is not None:
self.connect(self.phiMotor, 'stateChanged', self.phiMotorStateChanged)
self.connect(self.phiMotor, "positionChanged", self.phi_motor_position_changed)
else:
logging.getLogger("HWR").error('MiniDiff: phi motor is not defined in minidiff equipment %s', str(self.name()))
if self.sampleXMotor is not None:
self.connect(self.sampleXMotor, 'stateChanged', self.sampleX_motor_state_changed)
self.connect(self.sampleXMotor, 'positionChanged', self.sampleX_motor_moved)
self.connect(self.sampleXMotor, "positionChanged", self.emit_diffractometer_moved)
else:
logging.getLogger("HWR").error('MiniDiff: Sampx motor is not defined')
if self.sampleYMotor is not None:
self.connect(self.sampleYMotor, 'stateChanged', self.sampleY_motor_state_changed)
self.connect(self.sampleYMotor, 'positionChanged', self.sampleY_motor_moved)
self.connect(self.sampleYMotor, "positionChanged", self.emit_diffractometer_moved)
else:
logging.getLogger("HWR").error('MiniDiff: Sampx motor is not defined')
self.beam_info_hwobj = HardwareRepository.HardwareRepository().\
getHardwareObject(self.getProperty("beam_info"))
if self.beam_info_hwobj is not None:
self.connect(self.beam_info_hwobj, 'beamPosChanged', self.beam_position_changed)
else:
logging.getLogger("HWR").debug('Minidiff: Beaminfo is not defined')
takeSnapshots = self.take_snapshots
self.getCentringStatus = self.get_centring_status
self.reversing_rotation = self.getProperty("reversingRotation")
try:
self.grid_direction = eval(self.getProperty("gridDirection"))
except:
self.grid_direction = {"fast": (0, 1), "slow": (1, 0)}
logging.getLogger("HWR").warning('MiniDiff: Grid direction is not defined. Using default.')
try:
self.current_phase = "Transfer"
self.phase_list = eval(self.getProperty("phaseList"))
self.head_type = self.getProperty("headType")
except:
self.phase_list = []
def set_drawing(self, drawing):
self._drawing = drawing
def use_sample_changer(self):
return True
def in_plate_mode(self):
#TODO head detection should be used to detect if in plate mode
return False
def toggle_fast_shutter(self):
self.fast_shutter_is_open = not self.fast_shutter_is_open
self.emit('minidiffShutterStateChanged', (self.fast_shutter_is_open, ))
def get_head_type(self):
return self.head_type
def get_grid_direction(self):
return self.grid_direction
def is_reversing_rotation(self):
return self.reversing_rotation == True
def phiMotorStateChanged(self,state):
self.emit('phiMotorStateChanged', (state, ))
self.emit('minidiffStateChanged', (state,))
def phi_motor_position_changed(self, position):
self.current_positions_dict["phi"] = position
self.emit('minidiffStateChanged', ("ready",))
def sampleX_motor_state_changed(self, state):
"""
Descript. :
"""
self.emit('sampxMotorStateChanged', (state, ))
self.emit('minidiffStateChanged', (state, ))
def sampleY_motor_state_changed(self, state):
"""
Descript. :
"""
self.emit('sampyMotorStateChanged', (state, ))
self.emit('minidiffStateChanged', (state, ))
def sampleX_motor_moved(self, pos):
"""
Descript. :
"""
self.current_positions_dict["sampx"] = pos
if time.time() - self.centring_time > 1.0:
self.invalidate_centring()
def sampleY_motor_moved(self, pos):
"""
Descript. :
"""
self.current_positions_dict["sampy"] = pos
if time.time() - self.centring_time > 1.0:
self.invalidate_centring()
def getStatus(self):
"""
Descript. :
"""
return "ready"
def manual_centring(self):
"""
Descript. :
"""
self.user_clicked_event = AsyncResult()
x, y = self.user_clicked_event.get()
last_centred_position[0] = x
last_centred_position[1] = y
random_num = random.random()
centred_pos_dir = {'phiy': random_num, 'phiz': random_num * 2,
'sampx': random_num * 3, 'sampy': random_num * 4,
'zoom': 8.53, 'phi': 311.1, 'focus': -0.42,
'kappa': 0.0, 'kappa_phi': 0.0}
return centred_pos_dir
def set_sample_info(self, sample_info):
"""
Descript. :
"""
self.current_sample_info = sample_info
def emit_diffractometer_moved(self, *args):
"""
Descript. :
"""
self.emit("diffractometerMoved", ())
def isReady(self):
"""
Descript. :
"""
return True
def isValid(self):
"""
Descript. :
"""
return True
def equipmentReady(self):
"""
Descript. :
"""
self.emit('minidiffReady', ())
def equipmentNotReady(self):
"""
Descript. :
"""
self.emit('minidiffNotReady', ())
def invalidate_centring(self):
"""
Descript. :
"""
if self.current_centring_procedure is None and self.centring_status["valid"]:
self.centring_status = {"valid":False}
self.emitProgressMessage("")
self.emit('centringInvalid', ())
def get_available_centring_methods(self):
"""
Descript. :
"""
return self.centring_methods.keys()
def get_calibration_data(self, offset):
"""
Descript. :
"""
#return (1.0 / self.x_calib, 1.0 / self.y_calib)
return (1.0 / self.x_calib, 1.0 / self.y_calib)
def get_pixels_per_mm(self):
"""
Descript. :
"""
return (self.pixels_per_mm_x, self.pixels_per_mm_y)
def refresh_omega_reference_position(self):
"""
Descript. :
"""
return
def get_omega_axis_position(self):
"""
Descript. :
"""
return self.current_positions_dict.get("phi")
def get_positions(self):
"""
Descript. :
"""
return self.current_positions_dict
def get_current_positions_dict(self):
"""
Descript. :
"""
return self.current_positions_dict
def beam_position_changed(self, value):
"""
Descript. :
"""
self.beam_position = value
def start_centring_method(self, method, sample_info = None):
"""
Descript. :
"""
if self.current_centring_method is not None:
logging.getLogger("HWR").error("already in centring method %s" %\
self.current_centring_method)
return
curr_time = time.strftime("%Y-%m-%d %H:%M:%S")
self.centring_status = {"valid": False, "startTime": curr_time}
self.emit_centring_started(method)
try:
fun = self.centring_methods[method]
except KeyError, diag:
logging.getLogger("HWR").error("unknown centring method (%s)" % \
str(diag))
self.emit_centring_failed()
else:
def listen():
waiter = AsyncResult()
waiters.append(waiter)
return "($('body').html('%s'));" % waiter.get()
class TestRemoteEndpoint(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
Start fake terrestrial components and add cleanup.
Start terrestrial server and retrieve port.
Set internal variables.
Start container.
Start deployment.
Start container agent.
Spawn remote endpoint process.
Create remote endpoint client and retrieve remote server port.
Create event publisher.
"""
self._terrestrial_server = R3PCServer(self.consume_req, self.terrestrial_server_close)
self._terrestrial_client = R3PCClient(self.consume_ack, self.terrestrial_client_close)
self.addCleanup(self._terrestrial_server.stop)
self.addCleanup(self._terrestrial_client.stop)
self._other_port = self._terrestrial_server.start('*', 0)
log.debug('Terrestrial server binding to *:%i', self._other_port)
self._other_host = 'localhost'
self._platform_resource_id = 'abc123'
self._resource_id = 'fake_id'
self._no_requests = 10
self._requests_sent = {}
self._results_recv = {}
self._no_telem_events = 0
self._done_evt = AsyncResult()
self._done_telem_evts = AsyncResult()
self._cmd_tx_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Create agent config.
endpoint_config = {
'other_host' : self._other_host,
'other_port' : self._other_port,
'this_port' : 0,
'platform_resource_id' : self._platform_resource_id
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
re_pid = container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(re_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=re_pid)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._this_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._this_port)
# Start the event publisher.
self._event_publisher = EventPublisher()
######################################################################
# Helpers.
######################################################################
def on_link_up(self):
"""
Called by a test to simulate turning the link on.
"""
log.debug('Terrestrial client connecting to localhost:%i.',
self._this_port)
self._terrestrial_client.start('localhost', self._this_port)
# Publish a link up event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status = TelemetryStatusType.AVAILABLE)
def on_link_down(self):
"""
Called by a test to simulate turning the link off.
"""
self._terrestrial_client.stop()
# Publish a link down event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_req(self, res):
"""
Consume a terrestrial request setting async event when necessary.
"""
command_id = res['command_id']
self._results_recv[command_id] = res
if len(self._results_recv) == self._no_requests:
self._done_evt.set()
def consume_ack(self, cmd):
"""
Consume terrestrial ack setting async event when necessary.
"""
self._requests_sent[cmd.command_id] = cmd
if len(self._requests_sent) == self._no_requests:
self._cmd_tx_evt.set()
def terrestrial_server_close(self):
"""
Callback when terrestrial server closes.
"""
pass
def terrestrial_client_close(self):
"""
Callback when terrestrial client closes.
"""
pass
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id=self._resource_id,
command=cmdstr,
args=['arg1', 23],
kwargs={'worktime':3},
command_id = str(uuid.uuid4()))
return cmd
def start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
######################################################################
# Tests.
######################################################################
def test_process_queued(self):
"""
test_process_queued
Test that queued commands are forwarded to and handled by
remote endpoint when link comes up.
"""
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_process_online(self):
"""
test_process_online
Test commands are forwarded and handled while link is up.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_terrestrial_late(self):
"""
test_terrestrial_late
Test queued commands are forwarded and handled by remote endpoint
when terrestrial side is late to come up.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Manually stop the terrestrial endpoint.
# This will cause it to be unavailable when commands are queued
# to simulate stability during asynchronous wake ups.
self._terrestrial_server.stop()
self._terrestrial_client.stop()
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Remote side awaits the terrestrial waking up.
gevent.sleep(3)
# Terrestrail endpoint eventually wakes up and starts transmitting.
self._terrestrial_client.start('localhost', self._this_port)
self._terrestrial_server.start('*', self._other_port)
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_service_commands(self):
"""
test_service_commands
Test that real service commands are handled by the remote endpoint.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='create',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = self._results_recv[cmd.command_id]['result']
# Confirm the results are valid.
"""
Result is a tuple of strings.
{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
'1-00a308c45fff459c7cda1db9a7314de6'],
'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
"""
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Read user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id]['result']
# Confirm the results are valid.
"""
Result is a user info object with the name set.
{'lcstate': 'DEPLOYED_AVAILABLE',
'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
'description': '',
'ts_updated': '1349213207638',
'type_': 'UserInfo',
'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
'_id': '27832d93f4cd4535a75ac75c06e00a7e',
'ts_created': '1349213207638',
'variables': [{'name': '', 'value': ''}],
'name': 'some_name'}
"""
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Update user object.
read_obj.name = 'some_other_name'
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='update',
args=[read_obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Read user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Delete user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='delete',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_resource_commands(self):
"""
test_resource_commands
Test that real resource commands are handled by the remote endpoint.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns agent state.
state = self._results_recv[cmd.command_id]['result']
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Ping agent via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='ping_agent',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns agent state.
ping = self._results_recv[cmd.command_id]['result']
self.assertIn('ping from InstrumentAgent', ping)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_bad_service_name_resource_id(self):
"""
test_bad_service_name_resource_id
Test for proper exception behavior when a bad service name or
resource id is used in a command forwarded to the remote endpoint.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='bogus_service',
command='create',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, NotFound)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id='bogus_resource_id',
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, NotFound)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_bad_commands(self):
"""
test_bad_commands
Test for correct exception behavior if a bad command name is forwarded
to a remote service or resource.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='what_the_flunk',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns BadRequest.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, BadRequest)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='what_the_flunk',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, BadRequest)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_resource_command_sequence(self):
"""
test_resource_command_sequence
Test for successful completion of a properly ordered sequence of
resource commands queued for forwarding to the remote endpoint.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# We execute a sequence of twelve consecutive events.
self._no_requests = 12
# Get agent state.
cmd1 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd1)
# Initialize agent.
cmd2 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.INITIALIZE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd2)
# Get agent state.
cmd3 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd3)
# Go active.
cmd4 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd4)
# Get agent state.
cmd5 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd5)
# Run.
cmd6 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.RUN)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd6)
# Get agent state.
cmd7 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd7)
# Acquire sample.
cmd8 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd8)
# Acquire sample
cmd9 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd9)
# Acquire sample.
cmd10 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd10)
# Reset.
cmd11 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.RESET)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd11)
# Get agent state.
cmd12 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd12)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Check results of command sequence.
"""
0ccf1e10-eeca-400d-aefe-f9d6888ec963 {'result': 'RESOURCE_AGENT_STATE_INACTIVE', 'command_id': '0ccf1e10-eeca-400d-aefe-f9d6888ec963'}
92531bdf-c2c8-4aa8-817d-5107c7311b37 {'result': <interface.objects.AgentCommandResult object at 0x10d7f11d0>, 'command_id': '92531bdf-c2c8-4aa8-817d-5107c7311b37'}
509934a1-5038-40d8-8014-591e2d8042b6 {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '509934a1-5038-40d8-8014-591e2d8042b6'}
88bacbb7-5366-4d27-9ecf-fff2bec34b2c {'result': <interface.objects.AgentCommandResult object at 0x10d389190>, 'command_id': '88bacbb7-5366-4d27-9ecf-fff2bec34b2c'}
f8b4d3fa-a249-439b-8bd4-ac212b6100aa {'result': <interface.objects.AgentCommandResult object at 0x10d3893d0>, 'command_id': 'f8b4d3fa-a249-439b-8bd4-ac212b6100aa'}
8ae98e39-fdb3-4218-ad8f-584620397d9f {'result': <interface.objects.AgentCommandResult object at 0x10d739990>, 'command_id': '8ae98e39-fdb3-4218-ad8f-584620397d9f'}
746364a1-c4c7-400f-96d4-ee36df5dc1a4 {'result': BadRequest('Execute argument "command" not set.',), 'command_id': '746364a1-c4c7-400f-96d4-ee36df5dc1a4'}
d516d3d9-e4f9-4ea5-80e0-34639a6377b5 {'result': <interface.objects.AgentCommandResult object at 0x10d3b2350>, 'command_id': 'd516d3d9-e4f9-4ea5-80e0-34639a6377b5'}
c7da03f5-59bc-420a-9e10-0a7794266599 {'result': 'RESOURCE_AGENT_STATE_IDLE', 'command_id': 'c7da03f5-59bc-420a-9e10-0a7794266599'}
678d870a-bf18-424a-afb0-f80ecf3277e2 {'result': <interface.objects.AgentCommandResult object at 0x10d739590>, 'command_id': '678d870a-bf18-424a-afb0-f80ecf3277e2'}
750c6a30-56eb-4535-99c2-a81fefab1b1f {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '750c6a30-56eb-4535-99c2-a81fefab1b1f'}
c17bd658-3775-4aa3-8844-02df70a0e3c0 {'result': 'RESOURCE_AGENT_STATE_UNINITIALIZED', 'command_id': 'c17bd658-3775-4aa3-8844-02df70a0e3c0'}
"""
# First result is a state string.
result1 = self._results_recv[cmd1.command_id]['result']
self.assertEqual(result1, ResourceAgentState.UNINITIALIZED)
# Second result is an empty AgentCommandResult.
result2 = self._results_recv[cmd2.command_id]['result']
# Third result is a state string.
result3 = self._results_recv[cmd3.command_id]['result']
self.assertEqual(result3, ResourceAgentState.INACTIVE)
# Fourth result is an empty AgentCommandResult.
result4 = self._results_recv[cmd4.command_id]['result']
# Fifth result is a state string.
result5 = self._results_recv[cmd5.command_id]['result']
self.assertEqual(result5, ResourceAgentState.IDLE)
# Sixth result is an empty AgentCommandResult.
result6 = self._results_recv[cmd6.command_id]['result']
# Seventh result is a state string.
result7 = self._results_recv[cmd7.command_id]['result']
self.assertEqual(result7, ResourceAgentState.COMMAND)
"""
{'raw': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
'stream_name': 'raw', 'pkt_format_id': 'JSON_Data',
'pkt_version': 1, '
values': [{'binary': True, 'value_id': 'raw',
'value': 'NzkuNDM3MywxNy4yMDU2NCwgNzYxLjg4NSwgICA2LjIxOTgsIDE1MDYuMzk3LCAwMSBGZWIgMjAwMSwgMDE6MDE6MDA='}],
'driver_timestamp': 3558286748.8039923},
'parsed': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data', 'pkt_version': 1,
'values': [{'value_id': 'temp', 'value': 79.4373},
{'value_id': 'conductivity', 'value': 17.20564},
{'value_id': 'pressure', 'value': 761.885}],
'driver_timestamp': 3558286748.8039923}}
"""
# Eigth result is an AgentCommandResult containing a sample.
result8 = self._results_recv[cmd8.command_id]['result']
self.assertTrue('parsed',result8.result )
# Ninth result is an AgentCommandResult containing a sample.
result9 = self._results_recv[cmd9.command_id]['result']
self.assertTrue('parsed',result9.result )
# Tenth result is an AgentCommandResult containing a sample.
result10 = self._results_recv[cmd10.command_id]['result']
self.assertTrue('parsed',result10.result )
# Eleventh result is an empty AgentCommandResult.
result11 = self._results_recv[cmd11.command_id]['result']
# Twelth result is a state string.
result12 = self._results_recv[cmd12.command_id]['result']
self.assertEqual(result1, ResourceAgentState.UNINITIALIZED)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
class AceClient(object):
def __init__(self,
clientcounter,
ace,
connect_timeout=5,
result_timeout=10):
# Telnet socket response buffer
self._recvbuffer = None
# AceEngine socket
self._socket = None
# AceEngine read result timeout
self._resulttimeout = result_timeout
# AceEngine product key
self._product_key = None
# Result (Created with AsyncResult() on call)
self._auth = AsyncResult()
# Result for START URL
self._url = AsyncResult()
# Response time from AceEngine to get URL or DATA
self._videotimeout = None
# Result for CID
self._cid = AsyncResult()
# Result fo LOADASYNC
self._loadasync = AsyncResult()
# Current STATUS
self._status = AsyncResult()
# Current EVENT
self._event = AsyncResult()
# Current STATE
self._state = AsyncResult()
# Current AUTH
self._gender = self._age = None
# Seekback seconds.
self._seekback = None
# Did we get START command again? For seekback.
self._started_again = Event()
# ClientCounter
self._clientcounter = clientcounter
# AceConfig.ace
self._ace = ace
try:
self._socket = Telnet(self._ace['aceHostIP'],
self._ace['aceAPIport'], connect_timeout)
logging.debug(
'Successfully connected to AceStream on {aceHostIP}:{aceAPIport}'
.format(**self._ace))
except:
errmsg = 'The are no alive AceStream Engines found!'
raise AceException(errmsg)
def destroy(self):
'''
AceClient Destructor
'''
# Send SHUTDOWN to AceEngine
try:
self._write(AceMessage.request.SHUTDOWN)
except:
pass # Ignore exceptions on destroy
finally:
self._clientcounter.idleAce = None
def reset(self):
'''
Reset initial values
'''
self._started_again.clear()
self._url.set()
self._loadasync.set()
self._cid.set()
self._status.set()
self._event.set()
self._state.set()
def _write(self, message):
try:
self._socket.write('%s\r\n' % message)
logging.debug('>>> %s' % message)
except gevent.socket.error:
raise AceException('Error writing data to AceEngine API port')
def aceInit(self,
gender=AceConst.SEX_MALE,
age=AceConst.AGE_25_34,
product_key=None,
videoseekback=0,
videotimeout=30):
self._gender = gender
self._age = age
self._product_key = product_key
self._seekback = videoseekback
self._videotimeout = videotimeout
self._started_again.clear()
# Spawning telnet data reader with recvbuffer read timeout (allowable STATE 0 (IDLE) time)
gevent.spawn(
wrap_errors((EOFError, gevent.socket.error), self._recvData),
self._videotimeout).link_exception(lambda x: logging.error(
'Error reading data from AceEngine API port'))
self._auth = AsyncResult()
self._write(AceMessage.request.HELLO) # Sending HELLOBG
try:
params = self._auth.get(timeout=self._resulttimeout)
except gevent.Timeout as t:
errmsg = 'Engine response time %s exceeded. HELLOTS not resived!' % t
raise AceException(errmsg)
if isinstance(params, dict):
self._write(
AceMessage.request.READY(params.get('key', ''),
self._product_key))
else:
self._auth.set(params)
try:
if self._auth.get(
timeout=self._resulttimeout
) == 'NOTREADY': # Get NOTREADY instead AUTH user_auth_level
errmsg = 'NOTREADY recived from AceEngine! Wrong acekey?'
raise AceException(errmsg)
except gevent.Timeout as t:
errmsg = 'Engine response time %s exceeded. AUTH not resived!' % t
raise AceException(errmsg)
if int(params.get('version_code',
0)) >= 3003600: # Display download_stopped massage
params_dict = {'use_stop_notifications': '1'}
self._write(AceMessage.request.SETOPTIONS(params_dict))
def START(self, command, paramsdict, acestreamtype):
'''
Start video method. Get url for play from AceEngine
'''
paramsdict['stream_type'] = ' '.join(
['{}={}'.format(k, v) for k, v in acestreamtype.items()])
self._url = AsyncResult()
self._write(AceMessage.request.START(command.upper(), paramsdict))
try:
return self._url.get(timeout=self._videotimeout)
except gevent.Timeout as t:
errmsg = 'START URL not received! Engine response time %s exceeded' % t
raise AceException(errmsg)
def STOP(self):
'''
Stop video method
'''
if self._state:
self._write(AceMessage.request.STOP)
def LOADASYNC(self, command, params, sessionid='0'):
self._loadasync = AsyncResult()
self._write(
AceMessage.request.LOADASYNC(command.upper(), sessionid, params))
try:
return self._loadasync.get(
timeout=self._resulttimeout) # Get _contentinfo json
except gevent.Timeout as t:
errmsg = 'Engine response %s time exceeded. LOADARESP not resived!' % t
raise AceException(errmsg)
def GETCONTENTINFO(self, command, value, sessionid='0'):
paramsdict = {
command: value,
'developer_id': '0',
'affiliate_id': '0',
'zone_id': '0'
}
return self.LOADASYNC(command, paramsdict, sessionid)
def GETCID(self, command, value):
contentinfo = self.GETCONTENTINFO(command, value)
if contentinfo['status'] in (1, 2):
paramsdict = {
'checksum': contentinfo['checksum'],
'infohash': contentinfo['infohash'],
'developer_id': '0',
'affiliate_id': '0',
'zone_id': '0'
}
self._cid = AsyncResult()
self._write(AceMessage.request.GETCID(paramsdict))
try:
return self._cid.get(timeout=self._resulttimeout)[2:] # ##CID
except gevent.Timeout as t:
errmsg = 'Engine response time %s exceeded. CID not resived!' % t
raise AceException(errmsg)
else:
errmsg = 'LOADASYNC returned error with message: %s' % contentinfo[
'message']
raise AceException(errmsg)
def GETINFOHASH(self, command, value, sessionid='0', idx=0):
contentinfo = self.GETCONTENTINFO(command, value, sessionid)
if contentinfo['status'] in (1, 2):
return contentinfo['infohash'], next(
iter([x[0] for x in contentinfo['files'] if x[1] == int(idx)]),
None)
elif contentinfo['status'] == 0:
errmsg = 'LOADASYNC returned status 0: The transport file does not contain audio/video files'
raise AceException(errmsg)
else:
errmsg = 'LOADASYNC returned error with message: %s' % contentinfo[
'message']
raise AceException(errmsg)
def _recvData(self, timeout=30):
'''
Data receiver method for greenlet
'''
while 1:
# Destroy socket connection if AceEngine STATE 0 (IDLE) and we didn't read anything from socket until Nsec
with gevent.Timeout(timeout, False):
try:
self._recvbuffer = self._socket.read_until('\r\n',
None).strip()
except gevent.Timeout:
self.destroy()
except gevent.socket.timeout:
pass
except:
raise
else:
logging.debug('<<< %s' % unquote(self._recvbuffer))
# Parsing everything only if the string is not empty
# HELLOTS
if self._recvbuffer.startswith('HELLOTS'):
#version=engine_version version_code=version_code key=request_key http_port=http_port
self._auth.set({
k: v
for k, v in (x.split('=')
for x in self._recvbuffer.split()
if '=' in x)
})
# NOTREADY
elif self._recvbuffer.startswith('NOTREADY'):
self._auth.set('NOTREADY')
# AUTH
elif self._recvbuffer.startswith('AUTH'):
self._auth.set(
self._recvbuffer.split()[1]) # user_auth_level
# START
elif self._recvbuffer.startswith('START'):
# url [ad=1 [interruptable=1]] [stream=1] [pos=position]
params = {
k: v
for k, v in (x.split('=')
for x in self._recvbuffer.split()
if '=' in x)
}
if not self._seekback or self._started_again.ready(
) or params.get('stream', '') is not '1':
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seekback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
self._url.set(
self._recvbuffer.split()[1]) # url for play
# LOADRESP
elif self._recvbuffer.startswith('LOADRESP'):
self._loadasync.set(
json.loads(
unquote(''.join(
self._recvbuffer.split()[2:]))))
# STATE
elif self._recvbuffer.startswith('STATE'):
self._state.set(self._recvbuffer.split()
[1]) # STATE state_id -> STATE_NAME
# STATUS
elif self._recvbuffer.startswith('STATUS'):
self._tempstatus = self._recvbuffer.split()[1]
stat = [self._tempstatus.split(';')[0].split(':')[1]
] # main:????
if self._tempstatus.startswith('main:idle'): pass
elif self._tempstatus.startswith('main:loading'): pass
elif self._tempstatus.startswith('main:starting'): pass
elif self._tempstatus.startswith('main:check'): pass
elif self._tempstatus.startswith('main:err'):
pass # err;error_id;error_message
elif self._tempstatus.startswith('main:dl'): #dl;
stat.extend(
map(int,
self._tempstatus.split(';')[1:]))
elif self._tempstatus.startswith(
'main:wait'): #wait;time;
stat.extend(
map(int,
self._tempstatus.split(';')[2:]))
elif self._tempstatus.startswith(
('main:prebuf', 'main:buf')): #buf;progress;time;
stat.extend(
map(int,
self._tempstatus.split(';')[3:]))
try:
self._status.set({
k: v
for k, v in zip(AceConst.STATUS, stat)
}) # dl, wait, buf, prebuf
except:
self._status.set(
{'status':
stat[0]}) # idle, loading, starting, check
# CID
elif self._recvbuffer.startswith('##'):
self._cid.set(self._recvbuffer)
# INFO
elif self._recvbuffer.startswith('INFO'):
pass
# EVENT
elif self._recvbuffer.startswith('EVENT'):
self._tempevent = self._recvbuffer.split()
if self._seekback and not self._started_again.ready(
) and 'livepos' in self._tempevent:
params = {
k: v
for k, v in (x.split('=')
for x in self._tempevent
if '=' in x)
}
self._write(
AceMessage.request.LIVESEEK(
int(params['last']) - self._seekback))
self._started_again.set()
elif 'getuserdata' in self._tempevent:
self._write(
AceMessage.request.USERDATA(
self._gender, self._age))
elif 'cansave' in self._tempevent:
pass
elif 'showurl' in self._tempevent:
pass
elif 'download_stopped' in self._tempevent:
pass
# PAUSE
elif self._recvbuffer.startswith('PAUSE'):
pass #self._write(AceMessage.request.EVENT('pause'))
# RESUME
elif self._recvbuffer.startswith('RESUME'):
pass #self._write(AceMessage.request.EVENT('play'))
# STOP
elif self._recvbuffer.startswith('STOP'):
pass #self._write(AceMessage.request.EVENT('stop'))
# SHUTDOWN
elif self._recvbuffer.startswith('SHUTDOWN'):
self._socket.close()
break
class Bucket(AsyncBucket):
def __init__(self, *args, **kwargs):
"""
This class is a 'GEvent'-optimized subclass of libcouchbase
which utilizes the underlying IOPS structures and the gevent
event primitives to efficiently utilize couroutine switching.
"""
super(Bucket, self).__init__(IOPS(), *args, **kwargs)
def _do_ctor_connect(self):
if self.connected:
return
self._connect()
self._evconn = AsyncResult()
self._conncb = self._on_connected
self._evconn.get()
self._evconn = None
def _on_connected(self, err):
if err:
self._evconn.set_exception(err)
else:
self._evconn.set(None)
def _waitwrap(self, cbasync):
cur_thread = getcurrent()
errback = lambda r, x, y, z: cur_thread.throw(x, y, z)
cbasync.set_callbacks(cur_thread.switch, errback)
try:
return get_hub().switch()
finally:
# Deregister callbacks to prevent another request on the same
# greenlet to get the result from this context.
cbasync.set_callbacks(dummy_callback, dummy_callback)
def _meth_factory(meth, name):
def ret(self, *args, **kwargs):
return self._waitwrap(meth(self, *args, **kwargs))
return ret
def _http_request(self, **kwargs):
res = super(Bucket, self)._http_request(**kwargs)
w = Waiter()
res.callback = lambda x: w.switch(x)
res.errback = lambda x, c, o, b: w.throw(c, o, b)
return w.get()
def query(self, *args, **kwargs):
kwargs['itercls'] = GView
return super(Bucket, self).query(*args, **kwargs)
def n1ql_query(self, query, *args, **kwargs):
kwargs['itercls'] = GN1QLRequest
return super(Bucket, self).n1ql_query(query, *args, **kwargs)
def _get_close_future(self):
ev = Event()
def _dtor_cb(*args):
ev.set()
self._dtorcb = _dtor_cb
return ev
locals().update(AsyncBucket._gen_memd_wrappers(_meth_factory))
class TransactionDummy(ATransaction):
ping_timeout = 5 # sec
result_timeout = 20 # sec
def __init__(self,
callback_url: str,
local_timeout: float,
ping_timeout=None,
result_timeout=None):
super().__init__(ObjectId())
self.callback_url = callback_url
self.local_timeout = local_timeout
self.ping_timeout = ping_timeout if ping_timeout is not None else TransactionDummy.ping_timeout # type: float
self.result_timeout = result_timeout if result_timeout is not None else TransactionDummy.result_timeout # type: float
self.key = sha256(
bytes(str(self.id) +
str(int(time.time() * 10**6) ^ randint(0, 2**20)),
encoding="utf-8")).hexdigest()
debug_SSE.event({
"event": "init",
"t": datetime.now(),
"data": {
"callback_url": self.callback_url,
"local_timeout": self.local_timeout * 1000,
"result_timeout": self.result_timeout * 1000,
"ping_timeout": self.ping_timeout * 1000,
"key": self.key,
"_id": self.id
}
}) # DEBUG init
self._ping = Event()
self.result = AsyncResult()
self.ping_timeout_thread_obj = None # type: Greenlet
self.result_thread_obj = None # type: Greenlet
@g_async
def _spawn(self):
self.ping_timeout_thread_obj = self.ping_timeout_thread(
) # THREAD:1, loop
# wait((self.ready_commit, self.fail), timeout=self.local_timeout) # BLOCK, timeout
# wait((self.commit, self.fail)) # BLOCK
@g_async
def ping_timeout_thread(self):
while not (self.done.ready() or self.fail.ready()):
debug_SSE.event({
"event": "wait_ping",
"t": datetime.now(),
"data": None
}) # DEBUG wait_ping
w = wait((self._ping, self.done, self.fail),
count=1,
timeout=self.ping_timeout * 2) # BLOCK, timeout
if not len(w):
debug_SSE.event({
"event": "fail",
"t": datetime.now(),
"data": "ping timeout"
}) # DEBUG ping timeout
self.fail.set() # EMIT(fail)
break
if self._ping.ready():
debug_SSE.event({
"event": "ping",
"t": datetime.now(),
"data": None
}) # DEBUG ping
self._ping.clear() # EMIT(-ping)
sleep()
def do_work(self, resource):
self.result_thread_obj = self.result_thread(resource) # THREAD:1
@g_async
def result_thread(self, resource):
sleep(self.result_timeout) # BLOCK, sleep
if not (self.ready_commit.ready() or self.fail.ready()):
self.result.set(resource) # EMIT(result)
self.ready_commit.set() # EMIT(ready_commit)
debug_SSE.event({
"event": "ready_commit",
"t": datetime.now(),
"data": None
}) # DEBUG ready_commit
data = {"key": self.key, "response": {"data": self.result.get()}}
rp = requests.put(self.callback_url,
headers={"Connection": "close"},
json=data,
timeout=5)
# else:
# raise Exception("error during work")
def ping(self) -> bool:
if not (self.fail.ready() or self.done.ready()):
self._ping.set() # EMIT(ping)
return True
return False
@g_async
def do_commit(self):
if not self.fail.ready():
if self.ready_commit.ready() and self.result.ready():
self.commit.set() # EMIT(ping)
debug_SSE.event({
"event": "commit",
"t": datetime.now(),
"data": None
}) # DEBUG commit
else:
raise Exception("Error during commit")
sleep(randint(self.ping_timeout - 2, self.ping_timeout + 2))
data = {"key": self.key, "done": True}
rp = requests.put(self.callback_url,
headers={"Connection": "close"},
json=data)
debug_SSE.event({
"event": "done",
"t": datetime.now(),
"data": None
}) # DEBUG done
@g_async
def do_rollback(self):
self.fail.set() # EMIT(fail)
debug_SSE.event({
"event": "rollback",
"t": datetime.now(),
"data": None
}) # DEBUG rollback
def _read_piece(self, fd, offset):
event = AsyncResult()
callback = partial(self._read_callback, event)
aio_read(fd, offset, self.chunk_size, callback)
return event.get()
class AceClient(object):
def __init__(self,
clientcounter,
ace,
connect_timeout=5,
result_timeout=10):
# Telnet socket response buffer
self._recvbuffer = None
# Telnet socket response buffer read timeout
self._recvbuffertimeout = 30
# AceEngine socket
self._socket = None
# ClientCounter
self._clientcounter = clientcounter
# AceEngine read result timeout
self._resulttimeout = float(result_timeout)
# AceEngine product key
self._product_key = None
# Result (Created with AsyncResult() on call)
self._auth = AsyncResult()
# Result for START URL
self._url = AsyncResult()
# Response time from AceEngine to get URL or DATA
self._videotimeout = None
# Result for CID
self._cid = AsyncResult()
# Result fo LOADASYNC
self._loadasync = AsyncResult()
# Current STATUS
self._status = AsyncResult()
# Current EVENT
self._event = AsyncResult()
# Current STATE
self._state = AsyncResult()
# Current AUTH
self._gender = None
self._age = None
# Seekback seconds.
self._seekback = None
# Did we get START command again? For seekback.
self._started_again = Event()
try:
self._socket = Telnet(ace['aceHostIP'], ace['aceAPIport'],
connect_timeout)
logging.debug('Successfully connected to AceStream on %s:%s' %
(ace['aceHostIP'], ace['aceAPIport']))
except:
errmsg = 'The are no alive AceStream Engines found!'
raise AceException(errmsg)
else:
gevent.spawn(self._recvData) # Spawning telnet data reader
def destroy(self):
'''
AceClient Destructor
'''
# Trying to disconnect
logging.debug('Destroying AceStream client.....')
try:
self._write(AceMessage.request.SHUTDOWN)
except:
pass # Ignore exceptions on destroy
def reset(self):
'''
Reset initial values
'''
self._started_again.clear()
self._url.set()
self._loadasync.set()
self._cid.set()
def _write(self, message):
try:
logging.debug('>>> %s' % message)
self._socket.write('%s\r\n' % message)
except gevent.socket.error as e:
raise AceException('Telnet exception at socket write %s' % repr(e))
def aceInit(self,
gender=AceConst.SEX_MALE,
age=AceConst.AGE_25_34,
product_key=None,
videoseekback=0,
videotimeout=30):
self._gender = gender
self._age = age
self._product_key = product_key
self._seekback = videoseekback
self._videotimeout = float(videotimeout)
self._started_again.clear()
self._auth = AsyncResult()
self._write(AceMessage.request.HELLO) # Sending HELLOBG
try:
params = self._auth.get(timeout=self._resulttimeout)
except gevent.Timeout:
errmsg = 'Engine response time %ssec exceeded. HELLOTS not resived!' % self._resulttimeout
raise AceException(errmsg)
return
self._auth = AsyncResult()
self._write(
AceMessage.request.READY(params.get('key', ''), self._product_key))
try:
if self._auth.get(
timeout=self._resulttimeout
) == 'NOTREADY': # Get NOTREADY instead AUTH user_auth_level
errmsg = 'NOTREADY recived from AceEngine! Wrong acekey?'
raise AceException(errmsg)
return
except gevent.Timeout:
errmsg = 'Engine response time %ssec exceeded. AUTH not resived!' % self._resulttimeout
raise AceException(errmsg)
if int(params.get('version_code',
0)) >= 3003600: # Display download_stopped massage
params_dict = {'use_stop_notifications': '1'}
self._write(AceMessage.request.SETOPTIONS(params_dict))
def START(self, command, paramsdict, acestreamtype):
'''
Start video method
Return the url provided by AceEngine
'''
paramsdict['stream_type'] = ' '.join(
['{}={}'.format(k, v) for k, v in acestreamtype.items()])
self._url = AsyncResult()
self._write(AceMessage.request.START(command.upper(), paramsdict))
try:
return self._url.get(
timeout=self._videotimeout) # Get url for play from AceEngine
except gevent.Timeout:
errmsg = 'Engine response time %ssec exceeded. START URL not resived!' % self._videotimeout
raise AceException(errmsg)
def STOP(self):
'''
Stop video method
'''
self._state = AsyncResult()
self._write(AceMessage.request.STOP)
try:
self._state.get(timeout=self._resulttimeout)
self._started_again.clear()
except gevent.Timeout:
errmsg = 'Engine response time %ssec exceeded. STATE 0 (IDLE) not resived!' % self._resulttimeout
raise AceException(errmsg)
def LOADASYNC(self, command, params):
self._loadasync = AsyncResult()
self._write(
AceMessage.request.LOADASYNC(command.upper(),
random.randint(1, 100000), params))
try:
return self._loadasync.get(
timeout=self._resulttimeout) # Get _contentinfo json
except gevent.Timeout:
errmsg = 'Engine response %ssec time exceeded. LOADARESP not resived!' % self._resulttimeout
raise AceException(errmsg)
def GETCONTENTINFO(self, command, value):
paramsdict = {
command: value,
'developer_id': '0',
'affiliate_id': '0',
'zone_id': '0'
}
return self.LOADASYNC(command, paramsdict)
def GETCID(self, command, value):
contentinfo = self.GETCONTENTINFO(command, value)
if contentinfo['status'] in (1, 2):
paramsdict = {
'checksum': contentinfo['checksum'],
'infohash': contentinfo['infohash'],
'developer_id': '0',
'affiliate_id': '0',
'zone_id': '0'
}
self._cid = AsyncResult()
self._write(AceMessage.request.GETCID(paramsdict))
try:
return self._cid.get(timeout=self._resulttimeout)[2:] # ##CID
except gevent.Timeout:
errmsg = 'Engine response time %ssec exceeded. CID not resived!' % self._resulttimeout
raise AceException(errmsg)
else:
errmsg = 'LOADASYNC returned error with message: %s' % contentinfo[
'message']
raise AceException(errmsg)
def GETINFOHASH(self, command, value, idx=0):
contentinfo = self.GETCONTENTINFO(command, value)
if contentinfo['status'] in (1, 2):
return contentinfo['infohash'], [
x[0] for x in contentinfo['files'] if x[1] == int(idx)
][0]
elif contentinfo['status'] == 0:
errmsg = 'LOADASYNC returned status 0: The transport file does not contain audio/video files'
raise AceException(errmsg)
else:
errmsg = 'LOADASYNC returned error with message: %s' % contentinfo[
'message']
raise AceException(errmsg)
def AceStreamReader(self, url, cid, req_headers=None):
'''
Get stream from AceEngine url and write it to client(s)
'''
logging.debug('Start StreamReader for url: %s' % url)
with requests.Session() as session:
if req_headers:
session.headers.update(req_headers)
logging.debug('Sending headers from client to AceEngine: %s' %
session.headers)
try:
self._write(AceMessage.request.EVENT('play'))
# AceEngine return link for HLS stream
if url.endswith('.m3u8'):
_used_chunks = []
while self._state.get(
timeout=self._resulttimeout)[0] in ('2', '3'):
for line in session.get(
url, stream=True,
timeout=(5, self._videotimeout)).iter_lines():
if line.startswith(b'download not found'): return
if line.startswith(
b'http://') and line not in _used_chunks:
self.RAWDataReader(
session.get(line,
stream=True,
timeout=(5,
self._videotimeout)),
self._clientcounter.getClientsList(cid))
_used_chunks.append(line)
if len(_used_chunks) > 15: _used_chunks.pop(0)
# AceStream return link for HTTP stream
else:
self.RAWDataReader(
session.get(url,
stream=True,
timeout=(5, self._videotimeout)),
self._clientcounter.getClientsList(cid))
except Exception as err:
clients = self._clientcounter.getClientsList(cid)
if clients:
logging.error('"%s" StreamReader error: %s' %
(clients[0].channelName, repr(err)))
gevent.wait([
gevent.spawn(self.write_chunk, c, b'', True)
for c in clients
]) #b'0\r\n\r\n' - send the chunked trailer
finally:
_used_chunks = None
def RAWDataReader(self, stream, clients):
for chunk in stream.iter_content(
chunk_size=1048576 if 'Content-Length' in
stream.headers else None):
if chunk:
gevent.wait([
gevent.spawn(self.write_chunk, c, chunk) for c in clients
])
def write_chunk(self, client, chunk, chunk_trailer=None):
try:
client.out.write(
b'%X\r\n%s\r\n' %
(len(chunk),
chunk)) if not client.transcoder else client.out.write(chunk)
except:
client.destroy() # Client disconected
if chunk_trailer: client.destroy()
def _recvData(self):
'''
Data receiver method for greenlet
'''
while 1:
try:
with gevent.timeout.Timeout(self._recvbuffertimeout):
self._recvbuffer = self._socket.read_until(
'\r\n', timeout=None).strip()
except EOFError as e:
# if the connection is closed and no cooked data is available.
raise AceException(
'Telnet exception at socket read. AceClient destroyed %s' %
repr(e))
return
# Ignore error occurs while reading blank lines from socket in STATE 0 (IDLE)
except gevent.socket.timeout:
pass
# SHUTDOWN socket connection if AceEngine STATE 0 (IDLE) and we didn't read anything from socket until Nsec
except gevent.timeout.Timeout:
self.destroy()
self._clientcounter.idleAce = None
else: # Parsing everything only if the string is not empty
logging.debug('<<< %s' %
requests.compat.unquote(self._recvbuffer))
# HELLOTS
if self._recvbuffer.startswith('HELLOTS'):
# version=engine_version version_code=version_code key=request_key http_port=http_port
self._auth.set({
k: v
for k, v in (x.split('=')
for x in self._recvbuffer.split()
if '=' in x)
})
# NOTREADY
elif self._recvbuffer.startswith('NOTREADY'):
self._auth.set('NOTREADY')
# AUTH
elif self._recvbuffer.startswith('AUTH'):
self._auth.set(
self._recvbuffer.split()[1]) # user_auth_level
# START
elif self._recvbuffer.startswith('START'):
# url [ad=1 [interruptable=1]] [stream=1] [pos=position]
params = {
k: v
for k, v in (x.split('=')
for x in self._recvbuffer.split()
if '=' in x)
}
if not self._seekback or self._started_again.ready(
) or params.get('stream', '') is not '1':
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seekback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
self._url.set(
self._recvbuffer.split()[1]) # url for play
# LOADRESP
elif self._recvbuffer.startswith('LOADRESP'):
self._loadasync.set(
requests.compat.json.loads(
requests.compat.unquote(''.join(
self._recvbuffer.split()[2:]))))
# STATE
elif self._recvbuffer.startswith(
'STATE'): # tuple of (state_id, time of appearance)
self._state.set(
(self._recvbuffer.split()[1], gevent.time.time()))
# STATUS
elif self._recvbuffer.startswith('STATUS'):
self._tempstatus = self._recvbuffer.split()[1]
if self._tempstatus.startswith('main:idle'): pass
elif self._tempstatus.startswith('main:loading'): pass
elif self._tempstatus.startswith('main:starting'): pass
elif self._tempstatus.startswith('main:check'): pass
elif self._tempstatus.startswith('main:wait'): pass
elif self._tempstatus.startswith(
('main:prebuf', 'main:buf')):
pass #progress;time
#values = list(map(int, self._tempstatus.split(';')[3:]))
#self._status.set({k: v for k, v in zip(AceConst.STATUS, values)})
elif self._tempstatus.startswith('main:dl'):
pass
#values = list(map(int, self._tempstatus.split(';')[1:]))
#self._status.set({k: v for k, v in zip(AceConst.STATUS, values)})
elif self._tempstatus.startswith(
'main:err'): # err;error_id;error_message
self._status.set_exception(
AceException('%s with message %s' %
(self._tempstatus.split(';')[0],
self._tempstatus.split(';')[2])))
# CID
elif self._recvbuffer.startswith('##'):
self._cid.set(self._recvbuffer)
# INFO
elif self._recvbuffer.startswith('INFO'):
pass
# EVENT
elif self._recvbuffer.startswith('EVENT'):
self._tempevent = self._recvbuffer.split()
if self._seekback and not self._started_again.ready(
) and 'livepos' in self._tempevent:
params = {
k: v
for k, v in (x.split('=') for x in self._tempevent
if '=' in x)
}
self._write(
AceMessage.request.LIVESEEK(
int(params['last']) - self._seekback))
self._started_again.set()
elif 'getuserdata' in self._tempevent:
self._write(
AceMessage.request.USERDATA(
self._gender, self._age))
elif 'cansave' in self._tempevent:
pass
elif 'showurl' in self._tempevent:
pass
elif 'download_stopped' in self._tempevent:
pass
# PAUSE
elif self._recvbuffer.startswith('PAUSE'):
self._write(AceMessage.request.EVENT('pause'))
# RESUME
elif self._recvbuffer.startswith('RESUME'):
self._write(AceMessage.request.EVENT('play'))
# STOP
elif self._recvbuffer.startswith('STOP'):
pass
# SHUTDOWN
elif self._recvbuffer.startswith('SHUTDOWN'):
self._socket.close()
logging.debug('AceClient destroyed')
return
finally:
gevent.sleep()
class AceClient(object):
def __init__(self, acehostslist, connect_timeout=5, result_timeout=10):
# Receive buffer
self._recvbuffer = None
# Stream URL
self._url = None
# Ace stream socket
self._socket = None
# Result timeout
self._resulttimeout = result_timeout
# Shutting down flag
self._shuttingDown = Event()
# Product key
self._product_key = None
# Current STATUS
self._status = None
# Current STATE
self._state = None
# Current video position
self._position = None
# Available video position (loaded data)
self._position_last = None
# Buffered video pieces
self._position_buf = None
# Current AUTH
self._auth = None
self._gender = None
self._age = None
# Result (Created with AsyncResult() on call)
self._result = AsyncResult()
self._authevent = Event()
# Result for getURL()
self._urlresult = AsyncResult()
# Result for GETCID()
self._cidresult = AsyncResult()
# Event for resuming from PAUSE
self._resumeevent = Event()
# Seekback seconds.
self._seekback = AceConfig.videoseekback
# Did we get START command again? For seekback.
self._started_again = False
self._idleSince = time.time()
self._streamReaderConnection = None
self._streamReaderState = None
self._lock = threading.Condition(threading.Lock())
self._streamReaderQueue = gevent.queue.Queue(
maxsize=AceConfig.readcachesize) # Ring buffer
self._engine_version_code = 0
# Logger
logger = logging.getLogger('AceClientimport tracebacknt_init')
# Try to connect AceStream engine
for AceEngine in acehostslist:
try:
self._socket = telnetlib.Telnet(AceEngine[0], AceEngine[1],
connect_timeout)
AceConfig.acehost, AceConfig.aceAPIport, AceConfig.aceHTTPport = AceEngine[
0], AceEngine[1], AceEngine[2]
logger.debug('Successfully connected to AceStream on %s:%d' %
(AceEngine[0], AceEngine[1]))
break
except:
logger.debug('The are no alive AceStream on %s:%d' %
(AceEngine[0], AceEngine[1]))
pass
# Spawning recvData greenlet
if self._socket:
gevent.spawn(self._recvData)
gevent.sleep()
else:
logger.error('The are no alive AceStream Engines found')
return
def destroy(self):
'''
AceClient Destructor
'''
logger = logging.getLogger('AceClient_destroy') # Logger
if self._shuttingDown.isSet():
return # Already in the middle of destroying
self._resumeevent.set(
) # We should resume video to prevent read greenlet deadlock
self._urlresult.set() # And to prevent getUrl deadlock
# Trying to disconnect
try:
logger.debug('Destroying AceStream client.....')
self._shuttingDown.set()
self._write(AceMessage.request.SHUTDOWN)
except:
pass # Ignore exceptions on destroy
finally:
self._shuttingDown.set()
def reset(self):
self._started_again = False
self._idleSince = time.time()
self._streamReaderState = None
def _write(self, message):
try:
logger = logging.getLogger('AceClient_write')
logger.debug('>>> %s' % message)
self._socket.write('%s\r\n' % message)
except EOFError as e:
raise AceException('Write error! %s' % repr(e))
def aceInit(self,
gender=AceConst.SEX_MALE,
age=AceConst.AGE_25_34,
product_key=AceConfig.acekey):
self._product_key = product_key
self._gender = gender
self._age = age
self._seekback = AceConfig.videoseekback
self._started_again = False
logger = logging.getLogger('AceClient_aceInit')
self._write(AceMessage.request.HELLO) # Sending HELLOBG
if not self._authevent.wait(self._resulttimeout):
errmsg = 'Authentication timeout during AceEngine init. Wrong key?' # HELLOTS not resived from engine
logger.error(errmsg)
raise AceException(errmsg)
return
if not self._auth:
errmsg = 'Authentication error during AceEngine init. Wrong key?'
logger.error(errmsg)
raise AceException(errmsg)
return
# Display download_stopped massage
if self._engine_version_code >= 3003600:
self._write(AceMessage.request.SETOPTIONS)
def _getResult(self):
try:
result = self._result.get(timeout=self._resulttimeout)
if not result:
raise AceException('Result not received from %s:%s' %
(AceConfig.acehost, AceConfig.aceAPIport))
except gevent.Timeout:
raise AceException('gevent_Timeout')
return result
def START(self, datatype, value, stream_type):
'''
Start video method
'''
if stream_type == 'hls' and self._engine_version_code >= 3010500:
stream_type = 'output_format=hls' + ' transcode_audio=' + str(AceConfig.transcode_audio) \
+ ' transcode_mp3=' + str(AceConfig.transcode_mp3) \
+ ' transcode_ac3=' + str(AceConfig.transcode_ac3) \
+ ' preferred_audio_language=' + AceConfig.preferred_audio_language
else:
stream_type = 'output_format=http'
self._urlresult = AsyncResult()
self._write(
AceMessage.request.START(datatype.upper(), value, stream_type))
self._getResult()
def STOP(self):
'''
Stop video method
'''
if self._state and self._state != '0':
self._result = AsyncResult()
self._write(AceMessage.request.STOP)
self._getResult()
def LOADASYNC(self, datatype, params):
self._result = AsyncResult()
self._write(
AceMessage.request.LOADASYNC(
datatype.upper(),
random.randint(1, AceConfig.maxconns * 10000), params))
return self._getResult()
def GETCONTENTINFO(self, datatype, value):
paramsdict = {
datatype: value,
'developer_id': '0',
'affiliate_id': '0',
'zone_id': '0'
}
return self.LOADASYNC(datatype, paramsdict)
def GETCID(self, datatype, url):
contentinfo = self.GETCONTENTINFO(datatype, url)
self._cidresult = AsyncResult()
self._write(
AceMessage.request.GETCID(contentinfo.get('checksum'),
contentinfo.get('infohash'), 0, 0, 0))
cid = self._cidresult.get(True, 5)
return '' if not cid or cid == '' else cid[2:]
def getUrl(self, timeout=30):
logger = logging.getLogger('AceClient_getURL') # Logger
try:
res = self._urlresult.get(timeout=timeout)
return res
except gevent.Timeout:
errmsg = 'Engine response time exceeded. GetURL timeout!'
logger.error(errmsg)
raise AceException(errmsg)
def startStreamReader(self, url, cid, counter, req_headers=None):
logger = logging.getLogger('StreamReader')
logger.debug('Open video stream: %s' % url)
self._streamReaderState = 1
transcoder = None
if 'range' in req_headers: del req_headers['range']
logger.debug('Get headers from client: %s' % req_headers)
with requests.get(url,
headers=req_headers,
stream=True,
timeout=(5, None)) as self._streamReaderConnection:
try:
if self._streamReaderConnection.status_code not in (200, 206):
logger.error('Failed to open video stream %s' % url)
return None
if url.endswith('.m3u8'):
self._streamReaderConnection.headers = {
'Content-Type': 'application/octet-stream',
'Connection': 'Keep-Alive',
'Keep-Alive': 'timeout=15, max=100'
}
popen_params = {
"bufsize": AceConfig.readchunksize,
"stdout": PIPE,
"stderr": None,
"shell": False
}
if AceConfig.osplatform == 'Windows':
ffmpeg_cmd = 'ffmpeg.exe '
CREATE_NO_WINDOW = 0x08000000
CREATE_NEW_PROCESS_GROUP = 0x00000200
DETACHED_PROCESS = 0x00000008
popen_params.update(creationflags=CREATE_NO_WINDOW
| DETACHED_PROCESS
| CREATE_NEW_PROCESS_GROUP)
else:
ffmpeg_cmd = 'ffmpeg '
ffmpeg_cmd += '-hwaccel auto -hide_banner -loglevel fatal -re -i %s -c copy -f mpegts -' % url
transcoder = Popen(ffmpeg_cmd.split(), **popen_params)
out = transcoder.stdout
logger.warning(
'HLS stream detected. Ffmpeg transcoding started')
else:
out = self._streamReaderConnection.raw
except requests.exceptions.RequestException:
logger.error('Failed to open video stream %s' % url)
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
else:
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
self.play_event()
while 1:
self.getPlayEvent(
) # Wait for PlayEvent (stop/resume sending data from AceEngine to streamReaderQueue)
clients = counter.getClients(cid)
try:
data = out.read(AceConfig.readchunksize)
except:
data = None
if data is not None and clients:
if self._streamReaderQueue.full():
self._streamReaderQueue.get()
self._streamReaderQueue.put(data)
for c in clients:
try:
c.queue.put(data, timeout=5)
except gevent.queue.Full: #Queue.Full client does not read data from buffer until 5sec - disconnect it
if len(clients) > 1:
logger.debug('Disconnecting client: %s' %
c.handler.clientip)
c.destroy()
elif counter.count(cid) == 0:
logger.debug(
'All clients disconnected - broadcast stoped')
break
else:
logger.warning('No data received - broadcast stoped')
counter.deleteAll(cid)
break
finally:
with self._lock:
self._streamReaderState = None
self._lock.notifyAll()
if transcoder:
try:
transcoder.kill()
logger.warning('Ffmpeg transcoding stoped')
except:
pass
def closeStreamReader(self):
logger = logging.getLogger('StreamReader')
c = self._streamReaderConnection
if c:
logger.debug('Close video stream: %s' % c.url)
c.close()
self._streamReaderConnection = None
self._streamReaderQueue.queue.clear()
def getPlayEvent(self, timeout=None):
'''
Blocking while in PAUSE, non-blocking while in RESUME
'''
return self._resumeevent.wait(timeout=timeout)
# EVENTS from client to AceEngine
def play_event(self):
self._write(AceMessage.request.PLAYEVENT)
def pause_event(self):
self._write(AceMessage.request.PAUSEEVENT)
def stop_event(self):
self._write(AceMessage.request.STOPEVENT)
# END EVENTS
def _recvData(self):
'''
Data receiver method for greenlet
'''
logger = logging.getLogger('AceClient_recvdata')
while 1:
gevent.sleep()
try:
self._recvbuffer = self._socket.read_until('\r\n').strip()
logger.debug(
'<<< %s' %
requests.compat.unquote(self._recvbuffer).decode('utf8'))
except:
# If something happened during read, abandon reader.
logger.error('Exception at socket read. AceClient destroyed')
if not self._shuttingDown.isSet(): self._shuttingDown.set()
return
else:
# Parsing everything only if the string is not empty
# HELLOTS
if self._recvbuffer.startswith(AceMessage.response.HELLO):
try:
params = {
k: v
for k, v in (x.split('=')
for x in self._recvbuffer.split()
if '=' in x)
}
except:
logger.error("Can't parse HELLOTS")
params = {}
if 'version_code' in params:
self._engine_version_code = params['version_code']
if 'key' in params:
self._write(
AceMessage.request.READY_key(
params['key'], self._product_key))
self._request_key = None
else:
self._write(AceMessage.request.READY_nokey)
# NOTREADY
elif self._recvbuffer.startswith(AceMessage.response.NOTREADY):
self._auth = None
self._authevent.set()
logger.error('AceEngine is not ready. Wrong auth?')
# AUTH
elif self._recvbuffer.startswith(AceMessage.response.AUTH):
try:
self._auth = self._recvbuffer.split()[1]
self._write(
AceMessage.request.USERDATA(
self._gender, self._age))
except:
pass
self._authevent.set()
# GETUSERDATA
elif self._recvbuffer.startswith(
AceMessage.response.GETUSERDATA):
raise AceException('You should init me first!')
# LOADRESP
elif self._recvbuffer.startswith(AceMessage.response.LOADRESP):
_contentinfo = json.loads(
requests.compat.unquote(' '.join(
self._recvbuffer.split()[2:])).decode('utf8'))
if _contentinfo.get('status') == 100:
logger.error(
'LOADASYNC returned error with message: %s' %
_contentinfo.get('message'))
self._result.set(False)
else:
self._result.set(_contentinfo)
# START
elif self._recvbuffer.startswith(AceMessage.response.START):
if not self._seekback or self._started_again or not self._recvbuffer.endswith(
' stream=1'):
# If seekback is disabled, we use link in first START command.
# If seekback is enabled, we wait for first START command and
# ignore it, then do seekback in first EVENT position command
# AceStream sends us STOP and START again with new link.
# We use only second link then.
try:
self._urlresult.set(self._recvbuffer.split()[1])
self._resumeevent.set()
except IndexError as e:
self._url = None
else:
logger.debug('START received. Waiting for %s.' %
AceMessage.response.LIVEPOS)
# STATE
elif self._recvbuffer.startswith(AceMessage.response.STATE):
self._state = self._recvbuffer.split()[1]
if self._state in ('0', '1'):
self._result.set(True) #idle, starting
elif self._state == '6':
self._result.set(False) # error
# STATUS
elif self._recvbuffer.startswith(AceMessage.response.STATUS):
self._status = self._recvbuffer.split()[1].split(';')
if 'main:err' in set(
self._status): # main:err;error_id;error_message
logger.error('%s with message %s' %
(self._status[0], self._status[2]))
self._result.set_exception(
AceException('%s with message %s' %
(self._status[0], self._status[2])))
self._urlresult.set_exception(
AceException('%s with message %s' %
(self._status[0], self._status[2])))
# LIVEPOS
elif self._recvbuffer.startswith(AceMessage.response.LIVEPOS):
if self._seekback and not self._started_again:
try:
params = {
k: v
for k, v in (x.split('=')
for x in self._recvbuffer.split()
if '=' in x)
}
self._write(
AceMessage.request.LIVESEEK(
int(params['last']) - self._seekback))
logger.debug('Seeking back.....')
self._started_again = True
except:
logger.error("Can't parse %s" %
AceMessage.response.LIVEPOS)
# CID
elif self._recvbuffer.startswith('##') or len(
self._recvbuffer) == 0:
self._cidresult.set(self._recvbuffer)
#DOWNLOADSTOP
elif self._recvbuffer.startswith(
AceMessage.response.DOWNLOADSTOP):
pass
# INFO
elif self._recvbuffer.startswith(AceMessage.response.INFO):
pass
# SHOWURL
elif self._recvbuffer.startswith(AceMessage.response.SHOWURL):
pass
# CANSAVE
elif self._recvbuffer.startswith(AceMessage.response.CANSAVE):
pass
# PAUSE
elif self._recvbuffer.startswith(AceMessage.response.PAUSE):
self.pause_event()
self._resumeevent.clear()
# RESUME
elif self._recvbuffer.startswith(AceMessage.response.RESUME):
self.play_event()
self._resumeevent.set()
# STOP
elif self._recvbuffer.startswith(AceMessage.response.STOP):
pass
# SHUTDOWN
elif self._recvbuffer.startswith(AceMessage.response.SHUTDOWN):
self._socket.get_socket().shutdown(SHUT_WR)
self._recvbuffer = self._socket.read_all()
self._socket.close()
logger.debug('AceClient destroyed')
return
def fetch_blocks(self, blockhashes_chain):
# fetch blocks (no parallelism here)
log_st.debug('fetching blocks', num=len(blockhashes_chain))
assert blockhashes_chain
blockhashes_chain.reverse() # oldest to youngest
num_blocks = len(blockhashes_chain)
num_fetched = 0
while blockhashes_chain:
blockhashes_batch = blockhashes_chain[:self.max_blocks_per_request]
t_blocks = []
# try with protos
protocols = self.synchronizer.protocols
if not protocols:
log_st.warn('no protocols available')
return self.exit(success=False)
for proto in protocols:
if proto.is_stopped:
continue
assert proto not in self.requests
# request
log_st.debug('requesting blocks', num=len(blockhashes_batch))
deferred = AsyncResult()
self.requests[proto] = deferred
proto.send_getblocks(*blockhashes_batch)
try:
t_blocks = deferred.get(
block=True, timeout=self.blocks_request_timeout)
except gevent.Timeout:
log_st.warn('getblocks timed out, trying next proto')
continue
finally:
del self.requests[proto]
if not t_blocks:
log_st.warn('empty getblocks reply, trying next proto')
continue
elif not isinstance(t_blocks[0], TransientBlock):
log_st.warn('received unexpected data',
data=repr(t_blocks))
t_blocks = []
continue
# we have results
if not [b.header.hash for b in t_blocks
] == blockhashes_batch[:len(t_blocks)]:
log_st.warn('received wrong blocks, should ban peer')
t_blocks = []
continue
break
# add received t_blocks
num_fetched += len(t_blocks)
log_st.debug('received blocks',
num=len(t_blocks),
num_fetched=num_fetched,
total=num_blocks,
missing=num_blocks - num_fetched)
if not t_blocks:
log_st.warn('failed to fetch blocks',
missing=len(blockhashes_chain))
return self.exit(success=False)
ts = time.time()
log_st.debug('adding blocks',
qsize=self.chainservice.block_queue.qsize())
for t_block in t_blocks:
assert t_block.header.hash == blockhashes_chain.pop(0)
self.chainservice.add_block(
t_block, proto) # this blocks if the queue is full
log_st.debug('adding blocks done', took=time.time() - ts)
# done
last_block = t_block
assert not len(blockhashes_chain)
assert last_block.header.hash == self.blockhash
log_st.debug('syncing finished')
# at this point blocks are not in the chain yet, but in the add_block queue
if self.chain_difficulty >= self.chain.head.chain_difficulty():
self.chainservice.broadcast_newblock(last_block,
self.chain_difficulty,
origin=proto)
self.exit(success=True)
def forward(self, *args, **kwargs):
"""
Forward a service method to the terrestrial endpoint
through the service interface.
"""
func_name = kwargs.pop('func_name')
try:
link = kwargs.pop('link')
except KeyError:
link = True
cid = ''
try:
remote_timeout = kwargs.pop('remote_timeout')
if not isinstance(remote_timeout, int):
remote_timeout = 0
elif remote_timeout < 0:
remote_timeout = 0
elif remote_timeout == 0:
pass
else:
cid = str(uuid.uuid4())
except KeyError:
remote_timeout = 0
cmd = IonObject('RemoteCommand',
resource_id=self._resource_id,
svc_name=self._svc_name,
command=func_name,
command_id=cid,
args=args,
kwargs=kwargs)
if remote_timeout == 0:
return self._te_client.enqueue_command(cmd, link)
else:
if self._resource_id:
origin = self._resource_id
elif self._svc_name:
origin = self._svc_name + self._xs_name
pending_cmd = cmd
async_result_evt = AsyncResult()
def result_callback(evt, *args, **kwargs):
"""
Callback for subscriber retrive blocking results.
"""
#global async_result_evt
if evt.type_ == 'RemoteCommandResult':
cmd = evt.command
if cmd.command_id == pending_cmd.command_id:
async_result_evt.set(cmd)
sub = EventSubscriber(event_type='RemoteCommandResult',
origin=origin,
callback=result_callback)
sub.start()
#self._pending_cmd = cmd
cmd = self._te_client.enqueue_command(cmd, link)
try:
result = async_result_evt.get(timeout=remote_timeout)
#self._pending_cmd = None
sub.stop()
except gevent.Timeout:
#self._pending_cmd = None
sub.stop()
raise Timeout('Timed out waiting for remote result.')
return result
