def prepare(self):
self.id = str(uuid.uuid4())
self.final_hyp = ""
self.worker_done = Condition()
self.user_id = self.request.headers.get("device-id", "none")
self.content_id = self.request.headers.get("content-id", "none")
logging.info("%s: OPEN: user='******', content='%s'" % (self.id, self.user_id, self.content_id))
self.worker = None
self.error_status = 0
self.error_message = None
#Waiter thread for final hypothesis:
# self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
try:
self.worker = self.application.available_workers.pop()
self.application.send_status_update()
logging.info("%s: Using worker %s" % (self.id, self.__str__()))
self.worker.set_client_socket(self)
content_type = self.request.headers.get("Content-Type", None)
if content_type:
content_type = content_type_to_caps(content_type)
logging.info("%s: Using content type: %s" % (self.id, content_type))
self.worker.write_message(json.dumps(dict(id=self.id, content_type=content_type, user_id=self.user_id, content_id=self.content_id)))
except KeyError:
logging.warn("%s: No worker available for client request" % self.id)
self.set_status(503)
self.finish("No workers available")
def get_data(cls, account, source_filter, limit=100, skip=0):
"""
Gathers card information from Google Sheets
GET https://spreadsheets.google.com/feeds/list/[spreadsheet]/[worksheet]/private/full
"""
if not account or not account.enabled:
raise ValueError('cannot gather information without an account')
client = AsyncHTTPClient()
if source_filter.spreadsheet is None:
raise ValueError('required parameter spreadsheet missing')
if source_filter.worksheet is None:
raise ValueError('required parameter worksheet missing')
uri = "https://docs.google.com/spreadsheets/d/{}/export?format=csv&gid={}".format(
source_filter.spreadsheet, source_filter.worksheet)
app_log.info("Start retrieval of worksheet {}/{} for {}".format(
source_filter.spreadsheet, source_filter.worksheet, account._id))
lock = Condition()
oauth_client = account.get_client()
uri, headers, body = oauth_client.add_token(uri)
req = HTTPRequest(uri,
headers=headers,
body=body,
streaming_callback=lambda c: cls.write(c))
client.fetch(req, callback=lambda r: lock.notify())
yield lock.wait(timeout=timedelta(seconds=MAXIMUM_REQ_TIME))
app_log.info("Finished retrieving worksheet for {}".format(
account._id))
def get_data(cls, account, source_filter, limit=100, skip=0):
"""
Gathers card information from Google Sheets
GET https://spreadsheets.google.com/feeds/list/[spreadsheet]/[worksheet]/private/full
"""
if not account or not account.enabled:
raise ValueError('cannot gather information without an account')
client = AsyncHTTPClient()
if source_filter.spreadsheet is None:
raise ValueError('required parameter spreadsheet missing')
if source_filter.worksheet is None:
raise ValueError('required parameter worksheet missing')
uri = "https://docs.google.com/spreadsheets/d/{}/export?format=csv&gid={}".format(
source_filter.spreadsheet, source_filter.worksheet
)
app_log.info(
"Start retrieval of worksheet {}/{} for {}".format(source_filter.spreadsheet, source_filter.worksheet,
account._id))
lock = Condition()
oauth_client = account.get_client()
uri, headers, body = oauth_client.add_token(uri)
req = HTTPRequest(uri, headers=headers, body=body, streaming_callback=lambda c: cls.write(c))
client.fetch(req, callback=lambda r: lock.notify())
yield lock.wait(timeout=timedelta(seconds=MAXIMUM_REQ_TIME))
app_log.info(
"Finished retrieving worksheet for {}".format(account._id))
def __init__(self, child, loop=None):
self.condition = Condition()
self.next = []
Stream.__init__(self, child, loop=loop)
self.loop.add_callback(self.cb)
class FlowControlWindow(object):
__slots__ = ['condition', 'value']
def __init__(self, initial_value=DEFAULT_WINDOW_SIZE):
self.condition = Condition()
self.value = initial_value
@gen.coroutine
def available(self, timeout=None):
if self.value > 0:
raise gen.Return(self.value)
yield self.condition.wait(timeout=timeout)
raise gen.Return(self.value)
def consume(self, n):
"""Tries to consume n from value"""
consumed = min(self.value, n)
self.value -= consumed
return consumed
def produce(self, n):
self.value += n
self.condition.notify_all()
def __init__ (self, device_server, stream, address):
self.recv_msg_cond = Condition()
self.recv_msg = {}
self.send_msg_sem = Semaphore(1)
self.pending_request_cnt = 0
self.device_server = device_server
self.stream = stream
self.address = address
self.stream.set_nodelay(True)
self.idle_time = 0;
self.killed = False
self.sn = ""
self.private_key = ""
self.node_id = 0
self.name = ""
self.iv = None
self.cipher = None
#self.state_waiters = []
#self.state_happened = []
self.event_waiters = []
self.event_happened = []
self.ota_ing = False
self.ota_notify_done_future = None
self.post_ota = False
self.online_status = True
def __init__(self, upstream, **kwargs):
self.condition = Condition()
self.next = []
Stream.__init__(self, upstream, ensure_io_loop=True, **kwargs)
self.loop.add_callback(self.cb)
def clear(self):
"""Reset this PeerGroup.
This closes all connections to all known peers and forgets about
these peers.
:returns:
A Future that resolves with a value of None when the operation
has finished
"""
if self._resetting:
# If someone else is already resetting the PeerGroup, just block
# on them to be finished.
yield self._reset_condition.wait()
raise gen.Return(None)
self._resetting = True
if self._reset_condition is None:
self._reset_condition = Condition()
try:
for peer in self._peers.values():
peer.close()
finally:
self._peers = {}
self._resetting = False
self._reset_condition.notify_all()
def __init__ (self, device_server, stream, address, conn_pool):
self.fw_version = 0.0
self.recv_msg_cond = Condition()
self.recv_msg = {}
self.send_msg_sem = Semaphore(1)
self.pending_request_cnt = 0
self.device_server = device_server
self.device_server_conn_pool = conn_pool
self.stream = stream
self.address = address
self.stream.set_nodelay(True)
self.stream.set_close_callback(self.on_close)
self.timeout_handler_onlinecheck = None
self.timeout_handler_offline = None
self.killed = False
self.is_junk = False
self.sn = ""
self.private_key = ""
self.node_id = ""
self.user_id = ""
self.iv = None
self.cipher_down = None
self.cipher_up = None
self.event_waiters = []
self.event_happened = []
self.ota_ing = False
self.ota_notify_done_future = None
self.post_ota = False
self.online_status = True
async def refresh(self, fetch_packages=True):
# TODO: Use python-apt python lib rather than command line for updates
if self.refresh_condition is None:
self.refresh_condition = Condition()
else:
self.refresh_condition.wait()
return
try:
if fetch_packages:
await self.execute_cmd(f"{APT_CMD} update",
timeout=300.,
retries=3)
res = await self.execute_cmd_with_response("apt list --upgradable",
timeout=60.)
pkg_list = [p.strip() for p in res.split("\n") if p.strip()]
if pkg_list:
pkg_list = pkg_list[2:]
self.available_packages = [
p.split("/", maxsplit=1)[0] for p in pkg_list
]
pkg_list = "\n".join(self.available_packages)
logging.info(
f"Detected {len(self.available_packages)} package updates:"
f"\n{pkg_list}")
except Exception:
logging.exception("Error Refreshing System Packages")
self.init_evt.set()
self.refresh_condition.notify_all()
self.refresh_condition = None
class ImageManager():
def __init__(self):
# Image data
self._frame = None
# Flow control
self._condition = Condition()
def timestamp(self, img):
now = datetime.datetime.now()
stamp.stamp(img, (10, 10), str(now), size=20)
return img
def update_frame(self, frame):
self._frame = BytesIO(frame)
self.ready = True
@property
def frame(self):
return self._frame
@property
def ready(self):
return self._condition
@ready.setter
def ready(self, cond):
if cond is True:
self._condition.notify_all()
class gather(Stream):
def __init__(self, child, limit=10, client=None):
self.client = client or default_client()
self.queue = Queue(maxsize=limit)
self.condition = Condition()
Stream.__init__(self, child)
self.client.loop.add_callback(self.cb)
def update(self, x, who=None):
return self.queue.put(x)
@gen.coroutine
def cb(self):
while True:
x = yield self.queue.get()
L = [x]
while not self.queue.empty():
L.append(self.queue.get_nowait())
results = yield self.client._gather(L)
for x in results:
yield self.emit(x)
if self.queue.empty():
self.condition.notify_all()
@gen.coroutine
def flush(self):
while not self.queue.empty():
yield self.condition.wait()
class FrameGrabber(Thread):
'''Watch if a new frame is coming
'''
def __init__(self, messages):
super(FrameGrabber, self).__init__()
self.messages = messages
self.notifier = Condition()
self.frame = None
self.to_exit = Event()
def shutdown(self):
''' Stop the thread
'''
self.to_exit.set()
def run(self):
while not self.to_exit.is_set():
try:
message = self.messages.get(True, 1)
if message is None:
break
frame = message.get("ar_frame", None)
if frame is None:
continue
_, frame = cv2.imencode(".jpg", frame,
[cv2.IMWRITE_JPEG_QUALITY, 80])
frame = frame.tostring()
self.frame = frame
self.notifier.notify()
except Queue.Empty:
continue
class StreamingMJPEGOutput(StreamingOutput):
name = "mjpeg"
def __init__(self):
self.frame = None
self.buffer = BytesIO()
self.condition = Condition()
def write(self, buf):
app_log.debug(f"Received {len(buf)} bytes of data")
if buf.startswith(b'\xff\xd8'):
# New frame, copy the existing buffer's content and notify all
# clients it's available
self.buffer.truncate()
with self.condition:
self.frame = self.buffer.getvalue()
self.condition.notify_all()
self.buffer.seek(0)
return self.buffer.write(buf)
def flush(self):
app_log.debug("Received flush call")
self.buffer.truncate()
with self.condition:
self.frame = self.buffer.getvalue()
self.condition.notify_all()
self.buffer.seek(0)
def __init__(self, child, limit=10, client=None):
self.client = client or default_client()
self.queue = Queue(maxsize=limit)
self.condition = Condition()
Stream.__init__(self, child)
self.client.loop.add_callback(self.cb)
def __init__(self, application, request, **kwargs):
super(PingHandler, self).__init__(application, request, **kwargs)
self.callback_queue = None
self.condition = Condition()
self.response = None
self.corr_id = str(uuid.uuid4())
self.in_channel = self.application.get_app_component().rabbitmq[
'client'].channels['in']
def __init__(self, upstream, loop=None):
loop = loop or upstream.loop or IOLoop.current()
self.condition = Condition()
self.next = []
Stream.__init__(self, upstream, loop=loop)
self.loop.add_callback(self.cb)
class zip(Stream):
""" Combine streams together into a stream of tuples
We emit a new tuple once all streams have produce a new tuple.
See also
--------
combine_latest
zip_latest
"""
_graphviz_orientation = 270
_graphviz_shape = 'triangle'
def __init__(self, *upstreams, **kwargs):
self.maxsize = kwargs.pop('maxsize', 10)
self.condition = Condition()
self.literals = [(i, val) for i, val in enumerate(upstreams)
if not isinstance(val, Stream)]
self.buffers = {
upstream: deque()
for upstream in upstreams if isinstance(upstream, Stream)
}
upstreams2 = [
upstream for upstream in upstreams if isinstance(upstream, Stream)
]
Stream.__init__(self, upstreams=upstreams2, **kwargs)
def pack_literals(self, tup):
""" Fill buffers for literals whenever we empty them """
inp = list(tup)[::-1]
out = []
for i, val in self.literals:
while len(out) < i:
out.append(inp.pop())
out.append(val)
while inp:
out.append(inp.pop())
return tuple(out)
def update(self, x, who=None):
L = self.buffers[who] # get buffer for stream
L.append(x)
if len(L) == 1 and all(self.buffers.values()):
tup = tuple(self.buffers[up][0] for up in self.upstreams)
for buf in self.buffers.values():
buf.popleft()
self.condition.notify_all()
if self.literals:
tup = self.pack_literals(tup)
return self._emit(tup)
elif len(L) > self.maxsize:
return self.condition.wait()
def __init__(self, address):
"""
@brief Construct new instance
"""
self._address = address
self._ioloop = IOLoop.current()
self._stop_event = Event()
self._is_stopped = Condition()
self._socket = None
def on_server_loaded(server_context):
messages['workers'] = {'interval': 1000,
'deque': deque(maxlen=1000),
'times': deque(maxlen=1000),
'condition': Condition()}
server_context.add_periodic_callback(lambda: http_get('workers'), 1000)
messages['tasks'] = {'interval': 100,
'deque': deque(maxlen=1000),
'times': deque(maxlen=1000),
'condition': Condition()}
server_context.add_periodic_callback(lambda: http_get('tasks'), 100)
def __init__(self, buf=None, auto_close=True):
"""In-Memory based stream
:param buf: the buffer for the in memory stream
"""
self._stream = deque()
if buf:
self._stream.append(buf)
self.state = StreamState.init
self._condition = Condition()
self.auto_close = auto_close
self.exception = None
async def refresh(self):
if self.refresh_condition is None:
self.refresh_condition = Condition()
else:
self.refresh_condition.wait()
return
try:
await self._check_version()
except Exception:
logging.exception("Error Refreshing git state")
self.init_evt.set()
self.refresh_condition.notify_all()
self.refresh_condition = None
def __init__(self, *upstreams, **kwargs):
self.maxsize = kwargs.pop('maxsize', 10)
self.condition = Condition()
self.literals = [(i, val) for i, val in enumerate(upstreams)
if not isinstance(val, Stream)]
self.buffers = {upstream: deque()
for upstream in upstreams
if isinstance(upstream, Stream)}
upstreams2 = [upstream for upstream in upstreams if isinstance(upstream, Stream)]
Stream.__init__(self, upstreams=upstreams2, **kwargs)
class EventSource(object):
def __init__(self):
self.lock = Condition()
self.events = None
@tornado.gen.coroutine
def publish(self, events):
self.events = events
self.lock.notify_all()
@tornado.gen.coroutine
def wait(self):
yield self.lock.wait()
async def refresh(self):
if self.refresh_condition is None:
self.refresh_condition = Condition()
else:
self.refresh_condition.wait()
return
try:
self._get_local_version()
await self._get_remote_version()
except Exception:
logging.exception("Error Refreshing Client")
self.init_evt.set()
self.refresh_condition.notify_all()
self.refresh_condition = None
class Window(object):
def __init__(self, parent, stream_id, initial_window_size):
self.parent = parent
self.stream_id = stream_id
self.cond = Condition()
self.closed = False
self.size = initial_window_size
def close(self):
self.closed = True
self.cond.notify_all()
def _raise_error(self, code, message):
if self.parent is None:
raise ConnectionError(code, message)
else:
raise StreamError(self.stream_id, code)
def adjust(self, amount):
self.size += amount
if self.size > constants.MAX_WINDOW_SIZE:
self._raise_error(constants.ErrorCode.FLOW_CONTROL_ERROR,
"flow control window too large")
self.cond.notify_all()
def apply_window_update(self, frame):
try:
window_update, = struct.unpack('>I', frame.data)
except struct.error:
raise ConnectionError(constants.ErrorCode.FRAME_SIZE_ERROR,
"WINDOW_UPDATE incorrect size")
# strip reserved bit
window_update = window_update & 0x7fffffff
if window_update == 0:
self._raise_error(constants.ErrorCode.PROTOCOL_ERROR,
"window update must not be zero")
self.adjust(window_update)
@gen.coroutine
def consume(self, amount):
while not self.closed and self.size <= 0:
yield self.cond.wait()
if self.closed:
raise StreamClosedError()
if self.size < amount:
amount = self.size
if self.parent is not None:
amount = yield self.parent.consume(amount)
self.size -= amount
raise gen.Return(amount)
class zip(Stream):
""" Combine streams together into a stream of tuples
We emit a new tuple once all streams have produce a new tuple.
See also
--------
combine_latest
zip_latest
"""
_graphviz_orientation = 270
_graphviz_shape = 'triangle'
def __init__(self, *upstreams, **kwargs):
self.maxsize = kwargs.pop('maxsize', 10)
self.buffers = [deque() for _ in upstreams]
self.condition = Condition()
self.literals = [(i, val) for i, val in enumerate(upstreams)
if not isinstance(val, Stream)]
self.pack_literals()
self.buffers_by_stream = {
upstream: buffer
for upstream, buffer in builtins.zip(upstreams, self.buffers)
if isinstance(upstream, Stream)
}
upstreams2 = [
upstream for upstream in upstreams if isinstance(upstream, Stream)
]
Stream.__init__(self, upstreams=upstreams2, **kwargs)
def pack_literals(self):
""" Fill buffers for literals whenver we empty them """
for i, val in self.literals:
self.buffers[i].append(val)
def update(self, x, who=None):
L = self.buffers_by_stream[who] # get buffer for stream
L.append(x)
if len(L) == 1 and all(self.buffers):
tup = tuple(buf.popleft() for buf in self.buffers)
self.condition.notify_all()
if self.literals:
self.pack_literals()
return self._emit(tup)
elif len(L) > self.maxsize:
return self.condition.wait()
def __init__(self, *upstreams, **kwargs):
self.maxsize = kwargs.pop('maxsize', 10)
self.buffers = [deque() for _ in upstreams]
self.condition = Condition()
self.literals = [(i, val) for i, val in enumerate(upstreams)
if not isinstance(val, Stream)]
self.pack_literals()
self.buffers_by_stream = {upstream: buffer
for upstream, buffer in builtins.zip(upstreams, self.buffers)
if isinstance(upstream, Stream)}
upstreams2 = [upstream for upstream in upstreams if isinstance(upstream, Stream)]
Stream.__init__(self, upstreams=upstreams2, **kwargs)
def __init__(self, address, record_dest):
"""
@brief Construct new instance
If record_dest is not empty, create a folder named record_dest and record the received packages there.
"""
self._address = address
self.__record_dest = record_dest
if record_dest:
if not os.path.isdir(record_dest):
os.makedirs(record_dest)
self._ioloop = IOLoop.current()
self._stop_event = Event()
self._is_stopped = Condition()
self._socket = None
self.__last_package = 0
def clear(self):
"""Reset this PeerGroup.
This closes all connections to all known peers and forgets about
these peers.
:returns:
A Future that resolves with a value of None when the operation
has finished
"""
if self._resetting:
# If someone else is already resetting the PeerGroup, just block
# on them to be finished.
yield self._reset_condition.wait()
raise gen.Return(None)
self._resetting = True
if self._reset_condition is None:
self._reset_condition = Condition()
try:
for peer in self._peers.values():
peer.close()
finally:
self._peers = {}
self._resetting = False
self._reset_condition.notify_all()
class CounterCondition(object):
def __init__(self):
self.condition = Condition()
self.counter = 0
def increment(self, value=1):
self.counter += value
self.condition.notify_all()
@gen.coroutine
def wait_until(self, value):
while True:
yield self.condition.wait()
if self.counter >= value:
self.counter -= value
return
def __init__(self, session, home):
self.scrapper = Scrapper(session, home)
self.session = session
self.home = home
self.condition = Condition()
tornado.ioloop.IOLoop.current()\
.spawn_callback(self.main)
def __init__(self,
core_pool_size,
queue,
reject_handler,
coroutine_pool_name=None):
self._core_pool_size = core_pool_size
self._queue = queue
self._reject_handler = reject_handler
self._coroutine_pool_name = coroutine_pool_name or \
'tornado-coroutine-pool-%s' % uuid.uuid1().hex
self._core_coroutines_condition = Condition()
self._core_coroutines = {}
self._core_coroutines_wait_condition = Condition()
self._shutting_down = False
self._shuted_down = False
self._initialize_core_coroutines()
class latest(Stream):
""" Drop held-up data and emit the latest result
This allows you to skip intermediate elements in the stream if there is
some back pressure causing a slowdown. Use this when you only care about
the latest elements, and are willing to lose older data.
This passes through values without modification otherwise.
Examples
--------
>>> source.map(f).latest().map(g) # doctest: +SKIP
"""
_graphviz_shape = 'octagon'
def __init__(self, upstream, **kwargs):
self.condition = Condition()
self.next = []
Stream.__init__(self, upstream, ensure_io_loop=True, **kwargs)
self.loop.add_callback(self.cb)
def update(self, x, who=None):
self.next = [x]
self.loop.add_callback(self.condition.notify)
@gen.coroutine
def cb(self):
while True:
yield self.condition.wait()
[x] = self.next
yield self._emit(x)
def __init__ (self, device_server, stream, address):
self.fw_version = 0.0
self.recv_msg_cond = Condition()
self.recv_msg = {}
self.send_msg_sem = Semaphore(1)
self.pending_request_cnt = 0
self.device_server = device_server
self.stream = stream
self.address = address
self.stream.set_nodelay(True)
self.timeout_handler_onlinecheck = None
self.timeout_handler_offline = None
self.killed = False
self.sn = ""
self.private_key = ""
self.node_id = 0
self.iv = None
self.cipher_down = None
self.cipher_up = None
#self.state_waiters = []
#self.state_happened = []
self.event_waiters = []
self.event_happened = []
self.ota_ing = False
self.ota_notify_done_future = None
self.post_ota = False
self.online_status = True
def __init__(self, application, request, **kwargs):
super(PingHandler, self).__init__(application, request, **kwargs)
self.callback_queue = None
self.condition = Condition()
self.response = None
self.corr_id = str(uuid.uuid4())
self.in_channel = self.application.get_app_component().rabbitmq[
'client'].channels['in']
class PingHandler(firenado.tornadoweb.TornadoHandler):
def __init__(self, application, request, **kwargs):
super(PingHandler, self).__init__(application, request, **kwargs)
self.callback_queue = None
self.condition = Condition()
self.response = None
self.corr_id = str(uuid.uuid4())
self.in_channel = self.application.get_app_component().rabbitmq[
'client'].channels['in']
@gen.coroutine
def post(self):
self.in_channel.queue_declare(exclusive=True,
callback=self.on_request_queue_declared)
yield self.condition.wait()
self.write(self.response)
def on_request_queue_declared(self, response):
logger.info('Request temporary queue declared.')
self.callback_queue = response.method.queue
self.in_channel.basic_consume(self.on_response, no_ack=True,
queue=self.callback_queue)
self.in_channel.basic_publish(
exchange='',
routing_key='ping_rpc_queue',
properties=pika.BasicProperties(
reply_to=self.callback_queue,
correlation_id=self.corr_id,
),
body=self.request.body)
def on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = {
'data': body.decode("utf-8"),
'date': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
}
self.in_channel.queue_delete(queue=self.callback_queue)
self.condition.notify()
def __init__(self, buf=None, auto_close=True):
"""In-Memory based stream
:param buf: the buffer for the in memory stream
"""
self._stream = deque()
if buf:
self._stream.append(buf)
self.state = StreamState.init
self._condition = Condition()
self.auto_close = auto_close
self.exception = None
def test_future_close_callback(self):
# Regression test for interaction between the Future read interfaces
# and IOStream._maybe_add_error_listener.
rs, ws = yield self.make_iostream_pair()
closed = [False]
cond = Condition()
def close_callback():
closed[0] = True
cond.notify()
rs.set_close_callback(close_callback)
try:
ws.write(b'a')
res = yield rs.read_bytes(1)
self.assertEqual(res, b'a')
self.assertFalse(closed[0])
ws.close()
yield cond.wait()
self.assertTrue(closed[0])
finally:
rs.close()
ws.close()
class GameScrapperWorker(object):
def __init__(self, session, home):
self.scrapper = Scrapper(session, home)
self.session = session
self.home = home
self.condition = Condition()
tornado.ioloop.IOLoop.current()\
.spawn_callback(self.main)
@tornado.gen.coroutine
def main(self):
_log().info('scrapper sleeping')
yield self.condition.wait()
_log().info('scrapper woke up')
self.scrapper.scrap_missing()
tornado.ioloop.IOLoop.current()\
.spawn_callback(self.main)
raise tornado.gen.Return(False)
def __init__(self, tchannel, score_threshold=None):
"""Initializes a new PeerGroup.
:param tchannel:
TChannel used for communication by this PeerGroup
:param score_threshold:
A value in the ``[0, 1]`` range. If specifiede, this requires that
chosen peers havea score higher than this value when performing
requests.
"""
self.tchannel = tchannel
self._score_threshold = score_threshold
# Dictionary from hostport to Peer.
self._peers = {}
# Notified when a reset is performed. This allows multiple coroutines
# to block on the same reset.
self._resetting = False
self._reset_condition = Condition()
class ImporterWorker(object):
def __init__(self, session, home, scrapper):
self.session = session
self.home = home
self.condition = Condition()
self.scrapper = scrapper
tornado.ioloop.IOLoop.current()\
.spawn_callback(self.main)
@tornado.gen.coroutine
def main(self):
_log().info('importer sleeping')
yield self.condition.wait()
_log().info('importer woke up')
changed = True
while changed:
changed = yield self.work()
tornado.ioloop.IOLoop.current()\
.spawn_callback(self.main)
@tornado.gen.coroutine
def work(self):
changed = False
fi = slickbird.FileImporter(self.session, self.home)
for f in self.session.query(orm.Importerfile)\
.filter(orm.Importerfile.status == 'scanning'):
changed = True
r, status = fi.file_import(f.filename)
f.status = status
if status == 'moved':
self.scrapper.condition.notify()
yield tornado.gen.moment
self.session.commit()
self.scrapper.condition.notify()
raise tornado.gen.Return(changed)
class InMemStream(Stream):
def __init__(self, buf=None, auto_close=True):
"""In-Memory based stream
:param buf: the buffer for the in memory stream
"""
self._stream = deque()
if buf:
self._stream.append(buf)
self.state = StreamState.init
self._condition = Condition()
self.auto_close = auto_close
self.exception = None
def clone(self):
new_stream = InMemStream()
new_stream.state = self.state
new_stream.auto_close = self.auto_close
new_stream._stream = deque(self._stream)
return new_stream
def read(self):
def read_chunk(future):
if self.exception:
future.set_exception(self.exception)
return future
chunk = ""
while len(self._stream) and len(chunk) < common.MAX_PAYLOAD_SIZE:
chunk += self._stream.popleft()
future.set_result(chunk)
return future
read_future = tornado.concurrent.Future()
# We're not ready yet
if self.state != StreamState.completed and not len(self._stream):
wait_future = self._condition.wait()
wait_future.add_done_callback(
lambda f: f.exception() or read_chunk(read_future)
)
return read_future
return read_chunk(read_future)
def write(self, chunk):
if self.exception:
raise self.exception
if self.state == StreamState.completed:
raise UnexpectedError("Stream has been closed.")
if chunk:
self._stream.append(chunk)
self._condition.notify()
# This needs to return a future to match the async interface.
r = tornado.concurrent.Future()
r.set_result(None)
return r
def set_exception(self, exception):
self.exception = exception
self.close()
def close(self):
self.state = StreamState.completed
self._condition.notify()
def __init__(self):
# Image data
self._frame = None
# Flow control
self._condition = Condition()
def __init__(self, parent, stream_id, initial_window_size):
self.parent = parent
self.stream_id = stream_id
self.cond = Condition()
self.closed = False
self.size = initial_window_size
class DeviceConnection(object):
state_waiters = {}
state_happened = {}
def __init__ (self, device_server, stream, address):
self.fw_version = 0.0
self.recv_msg_cond = Condition()
self.recv_msg = {}
self.send_msg_sem = Semaphore(1)
self.pending_request_cnt = 0
self.device_server = device_server
self.stream = stream
self.address = address
self.stream.set_nodelay(True)
self.timeout_handler_onlinecheck = None
self.timeout_handler_offline = None
self.killed = False
self.sn = ""
self.private_key = ""
self.node_id = 0
self.iv = None
self.cipher_down = None
self.cipher_up = None
#self.state_waiters = []
#self.state_happened = []
self.event_waiters = []
self.event_happened = []
self.ota_ing = False
self.ota_notify_done_future = None
self.post_ota = False
self.online_status = True
@gen.coroutine
def secure_write (self, data):
if self.cipher_down:
cipher_text = self.cipher_down.encrypt(pad(data))
yield self.stream.write(cipher_text)
@gen.coroutine
def wait_hello (self):
try:
self._wait_hello_future = self.stream.read_bytes(64) #read 64bytes: 32bytes SN + 32bytes signature signed with private key
str1 = yield gen.with_timeout(timedelta(seconds=10), self._wait_hello_future,
io_loop=ioloop.IOLoop.current())
self.idle_time = 0 #reset the idle time counter
if len(str1) != 64:
self.stream.write("sorry\r\n")
yield gen.sleep(0.1)
self.kill_myself()
gen_log.debug("receive length != 64")
raise gen.Return(100) # length not match 64
if re.match(r'@\d\.\d', str1[0:4]):
#new version firmware
self._wait_hello_future = self.stream.read_bytes(4) #read another 4bytes
str2 = yield gen.with_timeout(timedelta(seconds=10), self._wait_hello_future, io_loop=ioloop.IOLoop.current())
self.idle_time = 0 #reset the idle time counter
if len(str2) != 4:
self.stream.write("sorry\r\n")
yield gen.sleep(0.1)
self.kill_myself()
gen_log.debug("receive length != 68")
raise gen.Return(100) # length not match 64
str1 += str2
self.fw_version = float(str1[1:4])
sn = str1[4:36]
sig = str1[36:68]
else:
#for version < 1.1
sn = str1[0:32]
sig = str1[32:64]
gen_log.info("accepted sn: %s @fw_version %.1f" % (sn, self.fw_version))
#query the sn from database
node = None
cur = self.device_server.cur
cur.execute('select * from nodes where node_sn="%s"'%sn)
rows = cur.fetchall()
if len(rows) > 0:
node = rows[0]
if not node:
self.stream.write("sorry\r\n")
yield gen.sleep(0.1)
self.kill_myself()
gen_log.info("node sn not found")
raise gen.Return(101) #node not found
key = node['private_key']
key = key.encode("ascii")
sig0 = hmac.new(key, msg=sn, digestmod=hashlib.sha256).digest()
gen_log.debug("sig: "+ binascii.hexlify(sig))
gen_log.debug("sig calc:"+ binascii.hexlify(sig0))
if sig0 == sig:
#send IV + AES Key
self.sn = sn
self.private_key = key
self.node_id = str(node['node_id'])
gen_log.info("valid hello packet from node %s" % self.node_id)
#remove the junk connection of the same sn
ioloop.IOLoop.current().add_callback(self.device_server.remove_junk_connection, self)
#init aes
self.iv = Random.new().read(AES.block_size)
self.cipher_down = AES.new(key, AES.MODE_CFB, self.iv, segment_size=128)
if self.fw_version > 1.0:
self.cipher_up = AES.new(key, AES.MODE_CFB, self.iv, segment_size=128)
else:
#for old version
self.cipher_up = self.cipher_down
cipher_text = self.iv + self.cipher_down.encrypt(pad("hello"))
gen_log.debug("cipher text: "+ cipher_text.encode('hex'))
self.stream.write(cipher_text)
raise gen.Return(0)
else:
self.stream.write("sorry\r\n")
yield gen.sleep(0.1)
self.kill_myself()
gen_log.error("signature not match: %s %s" % (sig, sig0))
raise gen.Return(102) #sig not match
except gen.TimeoutError:
self.kill_myself()
raise gen.Return(1)
except iostream.StreamClosedError:
self.kill_myself()
raise gen.Return(2)
#ioloop.IOLoop.current().add_future(self._serving_future, lambda future: future.result())
@gen.coroutine
def _loop_reading_input (self):
line = ""
piece = ""
while not self.killed:
msg = ""
try:
msg = yield self.stream.read_bytes(16)
msg = unpad(self.cipher_up.decrypt(msg))
line += msg
while line.find('\r\n') > -1:
#reset the timeout
if self.timeout_handler_onlinecheck:
ioloop.IOLoop.current().remove_timeout(self.timeout_handler_onlinecheck)
self.timeout_handler_onlinecheck = ioloop.IOLoop.current().call_later(60, self._online_check)
if self.timeout_handler_offline:
ioloop.IOLoop.current().remove_timeout(self.timeout_handler_offline)
self.timeout_handler_offline = ioloop.IOLoop.current().call_later(70, self._callback_when_offline)
index = line.find('\r\n')
piece = line[:index+2]
line = line[index+2:]
piece = piece.strip("\r\n")
if piece in ['##ALIVE##']:
gen_log.info('Node %s alive on %s channel!' % (self.node_id, self.device_server.role))
continue
json_obj = json.loads(piece)
gen_log.info('Node %s recv json on %s channel' % (self.node_id, self.device_server.role))
gen_log.debug('%s' % str(json_obj))
try:
state = None
event = None
if json_obj['msg_type'] == 'online_status':
if json_obj['msg'] in ['1',1,True]:
self.online_status = True
else:
self.online_status = False
continue
elif json_obj['msg_type'] == 'ota_trig_ack':
state = ('going', 'Node has been notified...')
self.ota_ing = True
if self.ota_notify_done_future:
self.ota_notify_done_future.set_result(1)
elif json_obj['msg_type'] == 'ota_status':
if json_obj['msg'] == 'started':
state = ('going', 'Downloading the firmware...')
else:
state = ('error', 'Failed to start the downloading.')
self.post_ota = True
elif json_obj['msg_type'] == 'ota_result':
if json_obj['msg'] == 'success':
state = ('done', 'Firmware updated.')
else:
state = ('error', 'Update failed. Please reboot the node and retry.')
self.post_ota = True
self.ota_ing = False
elif json_obj['msg_type'] == 'event':
event = json_obj
event.pop('msg_type')
gen_log.debug("state: ")
gen_log.debug(state)
gen_log.debug("event: ")
gen_log.debug(event)
if state:
#print self.state_waiters
#print self.state_happened
if self.state_waiters and self.sn in self.state_waiters and len(self.state_waiters[self.sn]) > 0:
f = self.state_waiters[self.sn].pop(0)
f.set_result(state)
if len(self.state_waiters[self.sn]) == 0:
del self.state_waiters[self.sn]
elif self.state_happened and self.sn in self.state_happened:
self.state_happened[self.sn].append(state)
else:
self.state_happened[self.sn] = [state]
elif event:
if len(self.event_waiters) == 0:
self.event_happened.append(event)
else:
for future in self.event_waiters:
future.set_result(event)
self.event_waiters = []
else:
self.recv_msg = json_obj
self.recv_msg_cond.notify()
yield gen.moment
except Exception,e:
gen_log.warn("Node %s: %s" % (self.node_id ,str(e)))
except iostream.StreamClosedError:
gen_log.error("StreamClosedError when reading from node %s" % self.node_id)
self.kill_myself()
return
except ValueError:
gen_log.warn("Node %s: %s can not be decoded into json" % (self.node_id, piece))
except Exception,e:
gen_log.error("Node %s: %s" % (self.node_id ,str(e)))
self.kill_myself()
return
yield gen.moment
class DeviceConnection(object):
state_waiters = {}
state_happened = {}
def __init__ (self, device_server, stream, address):
self.recv_msg_cond = Condition()
self.recv_msg = {}
self.send_msg_sem = Semaphore(1)
self.pending_request_cnt = 0
self.device_server = device_server
self.stream = stream
self.address = address
self.stream.set_nodelay(True)
self.idle_time = 0;
self.killed = False
self.sn = ""
self.private_key = ""
self.node_id = 0
self.name = ""
self.iv = None
self.cipher = None
#self.state_waiters = []
#self.state_happened = []
self.event_waiters = []
self.event_happened = []
self.ota_ing = False
self.ota_notify_done_future = None
self.post_ota = False
self.online_status = True
@gen.coroutine
def secure_write (self, data):
if self.cipher:
cipher_text = self.cipher.encrypt(pad(data))
yield self.stream.write(cipher_text)
@gen.coroutine
def wait_hello (self):
try:
self._wait_hello_future = self.stream.read_bytes(64) #read 64bytes: 32bytes SN + 32bytes signature signed with private key
str = yield gen.with_timeout(timedelta(seconds=10), self._wait_hello_future, io_loop=self.stream.io_loop)
self.idle_time = 0 #reset the idle time counter
if len(str) != 64:
self.stream.write("sorry\r\n")
yield gen.sleep(0.1)
self.kill_myself()
gen_log.debug("receive length != 64")
raise gen.Return(100) # length not match 64
sn = str[0:32]
sig = str[32:64]
gen_log.info("accepted sn: "+ sn)
#query the sn from database
node = None
for n in NODES_DATABASE:
if n['node_sn'] == sn:
node = n
break
if not node:
self.stream.write("sorry\r\n")
yield gen.sleep(0.1)
self.kill_myself()
gen_log.info("node sn not found")
raise gen.Return(101) #node not found
key = node['node_key']
key = key.encode("ascii")
sig0 = hmac.new(key, msg=sn, digestmod=hashlib.sha256).digest()
gen_log.debug("sig: "+ binascii.hexlify(sig))
gen_log.debug("sig calc:"+ binascii.hexlify(sig0))
if sig0 == sig:
#send IV + AES Key
self.sn = sn
self.private_key = key
self.node_id = node['node_sn']
self.name = node['name']
gen_log.info("valid hello packet from node %s" % self.name)
#remove the junk connection of the same sn
self.stream.io_loop.add_callback(self.device_server.remove_junk_connection, self)
#init aes
self.iv = Random.new().read(AES.block_size)
self.cipher = AES.new(key, AES.MODE_CFB, self.iv, segment_size=128)
cipher_text = self.iv + self.cipher.encrypt(pad("hello"))
gen_log.debug("cipher text: "+ cipher_text.encode('hex'))
self.stream.write(cipher_text)
raise gen.Return(0)
else:
self.stream.write("sorry\r\n")
yield gen.sleep(0.1)
self.kill_myself()
gen_log.error("signature not match: %s %s" % (sig, sig0))
raise gen.Return(102) #sig not match
except gen.TimeoutError:
self.kill_myself()
raise gen.Return(1)
except iostream.StreamClosedError:
self.kill_myself()
raise gen.Return(2)
#self.stream.io_loop.add_future(self._serving_future, lambda future: future.result())
@gen.coroutine
def _loop_reading_input (self):
line = ""
piece = ""
while not self.killed:
msg = ""
try:
msg = yield self.stream.read_bytes(16)
msg = unpad(self.cipher.decrypt(msg))
self.idle_time = 0 #reset the idle time counter
line += msg
while line.find('\r\n') > -1:
index = line.find('\r\n')
piece = line[:index+2]
line = line[index+2:]
piece = piece.strip("\r\n")
json_obj = json.loads(piece)
gen_log.info('Node %s: recv json: %s' % (self.name, str(json_obj)))
try:
state = None
event = None
if json_obj['msg_type'] == 'online_status':
if json_obj['msg'] in ['1',1,True]:
self.online_status = True
else:
self.online_status = False
continue
elif json_obj['msg_type'] == 'ota_trig_ack':
state = ('going', 'Node has been notified...')
self.ota_ing = True
if self.ota_notify_done_future:
self.ota_notify_done_future.set_result(1)
elif json_obj['msg_type'] == 'ota_status':
if json_obj['msg'] == 'started':
state = ('going', 'Downloading the firmware...')
else:
state = ('error', 'Failed to start the downloading.')
self.post_ota = True
elif json_obj['msg_type'] == 'ota_result':
if json_obj['msg'] == 'success':
state = ('done', 'Firmware updated.')
else:
state = ('error', 'Update failed. Please reboot the node and retry.')
self.post_ota = True
self.ota_ing = False
elif json_obj['msg_type'] == 'event':
event = json_obj
event.pop('msg_type')
gen_log.debug(state)
gen_log.debug(event)
if state:
#print self.state_waiters
#print self.state_happened
if self.state_waiters and self.sn in self.state_waiters and len(self.state_waiters[self.sn]) > 0:
f = self.state_waiters[self.sn].pop(0)
f.set_result(state)
if len(self.state_waiters[self.sn]) == 0:
del self.state_waiters[self.sn]
elif self.state_happened and self.sn in self.state_happened:
self.state_happened[self.sn].append(state)
else:
self.state_happened[self.sn] = [state]
elif event:
if len(self.event_waiters) == 0:
self.event_happened.append(event)
else:
for future in self.event_waiters:
future.set_result(event)
self.event_waiters = []
else:
self.recv_msg = json_obj
self.recv_msg_cond.notify()
yield gen.moment
except Exception,e:
gen_log.warn("Node %s: %s" % (self.name ,str(e)))
except iostream.StreamClosedError:
gen_log.error("StreamClosedError when reading from node %s" % self.name)
self.kill_myself()
return
except ValueError:
gen_log.warn("Node %s: %s can not be decoded into json" % (self.name, piece))
except Exception,e:
gen_log.error("Node %s: %s" % (self.name ,str(e)))
self.kill_myself()
return
yield gen.moment
def get_data(cls, account, source_filter, limit=100, skip=0):
source_filter = OneDriveFileFilter(source_filter)
if source_filter.file is None:
raise ValueError("required parameter file missing")
app_log.info("Starting to retrieve file for {}".format(account._id))
client = AsyncHTTPClient()
uri = "https://api.onedrive.com/v1.0/drive/items/{}/content".format(source_filter.file)
lock = Condition()
def crawl_url(url):
# some yummy regex
location_header_regex = re.compile(r"^Location:\s?(?P<uri>http:/{2}\S+)")
http_status_regex = re.compile(r"^HTTP/[\d\.]+\s(?P<status>\d+)")
receiving_file = False
# define our callbacks
def header_callback(header):
m = http_status_regex.match(header)
if m is not None:
# process our HTTP status header
status = m.group("status")
if int(status) == 200:
# if we're 200, we're receiving the file, not just a redirect
app_log.info("Receiving file {} for account {}".format(source_filter.file, account._id))
global receiving_file
receiving_file = True
m = location_header_regex.match(header)
if m is not None:
# process our location header
uri = m.group("uri")
# and grab _that_ url
app_log.info("Following redirect for file {}".format(source_filter.file))
crawl_url(uri)
def stream_callback(chunk):
# only dump out chunks that are of the file we're looking for
global receiving_file
if receiving_file:
app_log.info("Writing chunk of {}B".format(chunk.__len__()))
cls.write(chunk)
def on_completed(resp):
if 200 <= resp.code <= 299:
lock.notify()
oauth_client = account.get_client()
uri, headers, body = oauth_client.add_token(url)
req = HTTPRequest(
uri, headers=headers, body=body, header_callback=header_callback, streaming_callback=stream_callback
)
client.fetch(req, callback=on_completed)
crawl_url(uri)
# wait for us to complete
try:
yield lock.wait(timeout=timedelta(seconds=MAXIMUM_REQ_TIME))
app_log.info("File {} retrieved successfully".format(source_filter.file))
except gen.TimeoutError:
app_log.error("Request for file {} => {} timed out!".format(source_filter.file, account._id))
class PeerGroup(object):
"""A PeerGroup represents a collection of Peers.
Requests routed through a PeerGroup can be sent to either a specific peer
or a peer chosen at random.
"""
def __init__(self, tchannel, score_threshold=None):
"""Initializes a new PeerGroup.
:param tchannel:
TChannel used for communication by this PeerGroup
:param score_threshold:
A value in the ``[0, 1]`` range. If specifiede, this requires that
chosen peers havea score higher than this value when performing
requests.
"""
self.tchannel = tchannel
self._score_threshold = score_threshold
# Dictionary from hostport to Peer.
self._peers = {}
# Notified when a reset is performed. This allows multiple coroutines
# to block on the same reset.
self._resetting = False
# We'll create a Condition here later. We want to avoid it right now
# because it has a side-effect of scheduling some dummy work on the
# ioloop, which prevents us from forking (if you're into that).
self._reset_condition = None
def __str__(self):
return "<PeerGroup peers=%s>" % str(self._peers)
@gen.coroutine
def clear(self):
"""Reset this PeerGroup.
This closes all connections to all known peers and forgets about
these peers.
:returns:
A Future that resolves with a value of None when the operation
has finished
"""
if self._resetting:
# If someone else is already resetting the PeerGroup, just block
# on them to be finished.
yield self._reset_condition.wait()
raise gen.Return(None)
self._resetting = True
if self._reset_condition is None:
self._reset_condition = Condition()
try:
for peer in self._peers.values():
peer.close()
finally:
self._peers = {}
self._resetting = False
self._reset_condition.notify_all()
def get(self, hostport):
"""Get a Peer for the given destination.
A new Peer is added and returned if one does not already exist for the
given host-port. Otherwise, the existing Peer is returned.
"""
assert hostport, "hostport is required"
if hostport not in self._peers:
self._peers[hostport] = Peer(self.tchannel, hostport)
return self._peers[hostport]
def lookup(self, hostport):
"""Look up a Peer for the given host and port.
Returns None if a Peer for the given host-port does not exist.
"""
assert hostport, "hostport is required"
return self._peers.get(hostport, None)
def remove(self, hostport):
"""Delete the Peer for the given host port.
Does nothing if a matching Peer does not exist.
:returns: The removed Peer
"""
assert hostport, "hostport is required"
return self._peers.pop(hostport, None)
def add(self, peer):
"""Add an existing Peer to this group.
A peer for the given host-port must not already exist in the group.
"""
assert peer, "peer is required"
if isinstance(peer, basestring):
# Assume strings are host-ports
peer = Peer(self.tchannel, peer)
assert peer.hostport not in self._peers, "%s already has a peer" % peer.hostport
self._peers[peer.hostport] = peer
@property
def hosts(self):
"""Get all host-ports managed by this PeerGroup."""
return self._peers.keys()
@property
def peers(self):
"""Get all Peers managed by this PeerGroup."""
return self._peers.values()
def request(self, service, hostport=None, **kwargs):
"""Initiate a new request through this PeerGroup.
:param hostport:
If specified, requests will be sent to the specific host.
Otherwise, a known peer will be picked at random.
:param service:
Name of the service being called. Defaults to an empty string.
:param service_threshold:
If ``hostport`` was not specified, this specifies the score
threshold at or below which peers will be ignored.
"""
return PeerClientOperation(peer_group=self, service=service, hostport=hostport, **kwargs)
def choose(self, hostport=None, score_threshold=None, blacklist=None):
"""Choose a Peer that matches the given criteria.
The Peer with the highest score will be chosen.
:param hostport:
Specifies that the returned Peer must be for the given host-port.
Without this, all peers managed by this PeerGroup are
candidates. If this is present, ``score_threshold`` is ignored.
:param score_threshold:
If specified, Peers with a score equal to or below this will be
ignored. Defaults to the value specified when the PeerGroup was
initialized.
:param blacklist:
Peers on the blacklist won't be chosen.
:returns:
A Peer that matches all the requested criteria or None if no such
Peer was found.
"""
blacklist = blacklist or set()
if hostport:
return self.get(hostport)
score_threshold = score_threshold or self._score_threshold or 0
chosen_peer = None
chosen_score = 0
hosts = self._peers.viewkeys() - blacklist
for host in hosts:
peer = self.get(host)
score = peer.state.score()
if score <= score_threshold:
continue
if score > chosen_score:
chosen_peer = peer
chosen_score = score
return chosen_peer
