def prepare(self): self.id = str(uuid.uuid4()) self.final_hyp = "" self.worker_done = Condition() self.user_id = self.request.headers.get("device-id", "none") self.content_id = self.request.headers.get("content-id", "none") logging.info("%s: OPEN: user='******', content='%s'" % (self.id, self.user_id, self.content_id)) self.worker = None self.error_status = 0 self.error_message = None #Waiter thread for final hypothesis: # self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) try: self.worker = self.application.available_workers.pop() self.application.send_status_update() logging.info("%s: Using worker %s" % (self.id, self.__str__())) self.worker.set_client_socket(self) content_type = self.request.headers.get("Content-Type", None) if content_type: content_type = content_type_to_caps(content_type) logging.info("%s: Using content type: %s" % (self.id, content_type)) self.worker.write_message(json.dumps(dict(id=self.id, content_type=content_type, user_id=self.user_id, content_id=self.content_id))) except KeyError: logging.warn("%s: No worker available for client request" % self.id) self.set_status(503) self.finish("No workers available")
def get_data(cls, account, source_filter, limit=100, skip=0): """ Gathers card information from Google Sheets GET https://spreadsheets.google.com/feeds/list/[spreadsheet]/[worksheet]/private/full """ if not account or not account.enabled: raise ValueError('cannot gather information without an account') client = AsyncHTTPClient() if source_filter.spreadsheet is None: raise ValueError('required parameter spreadsheet missing') if source_filter.worksheet is None: raise ValueError('required parameter worksheet missing') uri = "https://docs.google.com/spreadsheets/d/{}/export?format=csv&gid={}".format( source_filter.spreadsheet, source_filter.worksheet) app_log.info("Start retrieval of worksheet {}/{} for {}".format( source_filter.spreadsheet, source_filter.worksheet, account._id)) lock = Condition() oauth_client = account.get_client() uri, headers, body = oauth_client.add_token(uri) req = HTTPRequest(uri, headers=headers, body=body, streaming_callback=lambda c: cls.write(c)) client.fetch(req, callback=lambda r: lock.notify()) yield lock.wait(timeout=timedelta(seconds=MAXIMUM_REQ_TIME)) app_log.info("Finished retrieving worksheet for {}".format( account._id))
def get_data(cls, account, source_filter, limit=100, skip=0): """ Gathers card information from Google Sheets GET https://spreadsheets.google.com/feeds/list/[spreadsheet]/[worksheet]/private/full """ if not account or not account.enabled: raise ValueError('cannot gather information without an account') client = AsyncHTTPClient() if source_filter.spreadsheet is None: raise ValueError('required parameter spreadsheet missing') if source_filter.worksheet is None: raise ValueError('required parameter worksheet missing') uri = "https://docs.google.com/spreadsheets/d/{}/export?format=csv&gid={}".format( source_filter.spreadsheet, source_filter.worksheet ) app_log.info( "Start retrieval of worksheet {}/{} for {}".format(source_filter.spreadsheet, source_filter.worksheet, account._id)) lock = Condition() oauth_client = account.get_client() uri, headers, body = oauth_client.add_token(uri) req = HTTPRequest(uri, headers=headers, body=body, streaming_callback=lambda c: cls.write(c)) client.fetch(req, callback=lambda r: lock.notify()) yield lock.wait(timeout=timedelta(seconds=MAXIMUM_REQ_TIME)) app_log.info( "Finished retrieving worksheet for {}".format(account._id))
def __init__(self, child, loop=None): self.condition = Condition() self.next = [] Stream.__init__(self, child, loop=loop) self.loop.add_callback(self.cb)
class FlowControlWindow(object): __slots__ = ['condition', 'value'] def __init__(self, initial_value=DEFAULT_WINDOW_SIZE): self.condition = Condition() self.value = initial_value @gen.coroutine def available(self, timeout=None): if self.value > 0: raise gen.Return(self.value) yield self.condition.wait(timeout=timeout) raise gen.Return(self.value) def consume(self, n): """Tries to consume n from value""" consumed = min(self.value, n) self.value -= consumed return consumed def produce(self, n): self.value += n self.condition.notify_all()
def __init__ (self, device_server, stream, address): self.recv_msg_cond = Condition() self.recv_msg = {} self.send_msg_sem = Semaphore(1) self.pending_request_cnt = 0 self.device_server = device_server self.stream = stream self.address = address self.stream.set_nodelay(True) self.idle_time = 0; self.killed = False self.sn = "" self.private_key = "" self.node_id = 0 self.name = "" self.iv = None self.cipher = None #self.state_waiters = [] #self.state_happened = [] self.event_waiters = [] self.event_happened = [] self.ota_ing = False self.ota_notify_done_future = None self.post_ota = False self.online_status = True
def __init__(self, upstream, **kwargs): self.condition = Condition() self.next = [] Stream.__init__(self, upstream, ensure_io_loop=True, **kwargs) self.loop.add_callback(self.cb)
def clear(self): """Reset this PeerGroup. This closes all connections to all known peers and forgets about these peers. :returns: A Future that resolves with a value of None when the operation has finished """ if self._resetting: # If someone else is already resetting the PeerGroup, just block # on them to be finished. yield self._reset_condition.wait() raise gen.Return(None) self._resetting = True if self._reset_condition is None: self._reset_condition = Condition() try: for peer in self._peers.values(): peer.close() finally: self._peers = {} self._resetting = False self._reset_condition.notify_all()
def __init__ (self, device_server, stream, address, conn_pool): self.fw_version = 0.0 self.recv_msg_cond = Condition() self.recv_msg = {} self.send_msg_sem = Semaphore(1) self.pending_request_cnt = 0 self.device_server = device_server self.device_server_conn_pool = conn_pool self.stream = stream self.address = address self.stream.set_nodelay(True) self.stream.set_close_callback(self.on_close) self.timeout_handler_onlinecheck = None self.timeout_handler_offline = None self.killed = False self.is_junk = False self.sn = "" self.private_key = "" self.node_id = "" self.user_id = "" self.iv = None self.cipher_down = None self.cipher_up = None self.event_waiters = [] self.event_happened = [] self.ota_ing = False self.ota_notify_done_future = None self.post_ota = False self.online_status = True
async def refresh(self, fetch_packages=True): # TODO: Use python-apt python lib rather than command line for updates if self.refresh_condition is None: self.refresh_condition = Condition() else: self.refresh_condition.wait() return try: if fetch_packages: await self.execute_cmd(f"{APT_CMD} update", timeout=300., retries=3) res = await self.execute_cmd_with_response("apt list --upgradable", timeout=60.) pkg_list = [p.strip() for p in res.split("\n") if p.strip()] if pkg_list: pkg_list = pkg_list[2:] self.available_packages = [ p.split("/", maxsplit=1)[0] for p in pkg_list ] pkg_list = "\n".join(self.available_packages) logging.info( f"Detected {len(self.available_packages)} package updates:" f"\n{pkg_list}") except Exception: logging.exception("Error Refreshing System Packages") self.init_evt.set() self.refresh_condition.notify_all() self.refresh_condition = None
class ImageManager(): def __init__(self): # Image data self._frame = None # Flow control self._condition = Condition() def timestamp(self, img): now = datetime.datetime.now() stamp.stamp(img, (10, 10), str(now), size=20) return img def update_frame(self, frame): self._frame = BytesIO(frame) self.ready = True @property def frame(self): return self._frame @property def ready(self): return self._condition @ready.setter def ready(self, cond): if cond is True: self._condition.notify_all()
class gather(Stream): def __init__(self, child, limit=10, client=None): self.client = client or default_client() self.queue = Queue(maxsize=limit) self.condition = Condition() Stream.__init__(self, child) self.client.loop.add_callback(self.cb) def update(self, x, who=None): return self.queue.put(x) @gen.coroutine def cb(self): while True: x = yield self.queue.get() L = [x] while not self.queue.empty(): L.append(self.queue.get_nowait()) results = yield self.client._gather(L) for x in results: yield self.emit(x) if self.queue.empty(): self.condition.notify_all() @gen.coroutine def flush(self): while not self.queue.empty(): yield self.condition.wait()
class FrameGrabber(Thread): '''Watch if a new frame is coming ''' def __init__(self, messages): super(FrameGrabber, self).__init__() self.messages = messages self.notifier = Condition() self.frame = None self.to_exit = Event() def shutdown(self): ''' Stop the thread ''' self.to_exit.set() def run(self): while not self.to_exit.is_set(): try: message = self.messages.get(True, 1) if message is None: break frame = message.get("ar_frame", None) if frame is None: continue _, frame = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 80]) frame = frame.tostring() self.frame = frame self.notifier.notify() except Queue.Empty: continue
class StreamingMJPEGOutput(StreamingOutput): name = "mjpeg" def __init__(self): self.frame = None self.buffer = BytesIO() self.condition = Condition() def write(self, buf): app_log.debug(f"Received {len(buf)} bytes of data") if buf.startswith(b'\xff\xd8'): # New frame, copy the existing buffer's content and notify all # clients it's available self.buffer.truncate() with self.condition: self.frame = self.buffer.getvalue() self.condition.notify_all() self.buffer.seek(0) return self.buffer.write(buf) def flush(self): app_log.debug("Received flush call") self.buffer.truncate() with self.condition: self.frame = self.buffer.getvalue() self.condition.notify_all() self.buffer.seek(0)
def __init__(self, child, limit=10, client=None): self.client = client or default_client() self.queue = Queue(maxsize=limit) self.condition = Condition() Stream.__init__(self, child) self.client.loop.add_callback(self.cb)
def __init__(self, application, request, **kwargs): super(PingHandler, self).__init__(application, request, **kwargs) self.callback_queue = None self.condition = Condition() self.response = None self.corr_id = str(uuid.uuid4()) self.in_channel = self.application.get_app_component().rabbitmq[ 'client'].channels['in']
def __init__(self, upstream, loop=None): loop = loop or upstream.loop or IOLoop.current() self.condition = Condition() self.next = [] Stream.__init__(self, upstream, loop=loop) self.loop.add_callback(self.cb)
class zip(Stream): """ Combine streams together into a stream of tuples We emit a new tuple once all streams have produce a new tuple. See also -------- combine_latest zip_latest """ _graphviz_orientation = 270 _graphviz_shape = 'triangle' def __init__(self, *upstreams, **kwargs): self.maxsize = kwargs.pop('maxsize', 10) self.condition = Condition() self.literals = [(i, val) for i, val in enumerate(upstreams) if not isinstance(val, Stream)] self.buffers = { upstream: deque() for upstream in upstreams if isinstance(upstream, Stream) } upstreams2 = [ upstream for upstream in upstreams if isinstance(upstream, Stream) ] Stream.__init__(self, upstreams=upstreams2, **kwargs) def pack_literals(self, tup): """ Fill buffers for literals whenever we empty them """ inp = list(tup)[::-1] out = [] for i, val in self.literals: while len(out) < i: out.append(inp.pop()) out.append(val) while inp: out.append(inp.pop()) return tuple(out) def update(self, x, who=None): L = self.buffers[who] # get buffer for stream L.append(x) if len(L) == 1 and all(self.buffers.values()): tup = tuple(self.buffers[up][0] for up in self.upstreams) for buf in self.buffers.values(): buf.popleft() self.condition.notify_all() if self.literals: tup = self.pack_literals(tup) return self._emit(tup) elif len(L) > self.maxsize: return self.condition.wait()
def __init__(self, address): """ @brief Construct new instance """ self._address = address self._ioloop = IOLoop.current() self._stop_event = Event() self._is_stopped = Condition() self._socket = None
def on_server_loaded(server_context): messages['workers'] = {'interval': 1000, 'deque': deque(maxlen=1000), 'times': deque(maxlen=1000), 'condition': Condition()} server_context.add_periodic_callback(lambda: http_get('workers'), 1000) messages['tasks'] = {'interval': 100, 'deque': deque(maxlen=1000), 'times': deque(maxlen=1000), 'condition': Condition()} server_context.add_periodic_callback(lambda: http_get('tasks'), 100)
def __init__(self, buf=None, auto_close=True): """In-Memory based stream :param buf: the buffer for the in memory stream """ self._stream = deque() if buf: self._stream.append(buf) self.state = StreamState.init self._condition = Condition() self.auto_close = auto_close self.exception = None
async def refresh(self): if self.refresh_condition is None: self.refresh_condition = Condition() else: self.refresh_condition.wait() return try: await self._check_version() except Exception: logging.exception("Error Refreshing git state") self.init_evt.set() self.refresh_condition.notify_all() self.refresh_condition = None
def __init__(self, *upstreams, **kwargs): self.maxsize = kwargs.pop('maxsize', 10) self.condition = Condition() self.literals = [(i, val) for i, val in enumerate(upstreams) if not isinstance(val, Stream)] self.buffers = {upstream: deque() for upstream in upstreams if isinstance(upstream, Stream)} upstreams2 = [upstream for upstream in upstreams if isinstance(upstream, Stream)] Stream.__init__(self, upstreams=upstreams2, **kwargs)
class EventSource(object): def __init__(self): self.lock = Condition() self.events = None @tornado.gen.coroutine def publish(self, events): self.events = events self.lock.notify_all() @tornado.gen.coroutine def wait(self): yield self.lock.wait()
async def refresh(self): if self.refresh_condition is None: self.refresh_condition = Condition() else: self.refresh_condition.wait() return try: self._get_local_version() await self._get_remote_version() except Exception: logging.exception("Error Refreshing Client") self.init_evt.set() self.refresh_condition.notify_all() self.refresh_condition = None
class Window(object): def __init__(self, parent, stream_id, initial_window_size): self.parent = parent self.stream_id = stream_id self.cond = Condition() self.closed = False self.size = initial_window_size def close(self): self.closed = True self.cond.notify_all() def _raise_error(self, code, message): if self.parent is None: raise ConnectionError(code, message) else: raise StreamError(self.stream_id, code) def adjust(self, amount): self.size += amount if self.size > constants.MAX_WINDOW_SIZE: self._raise_error(constants.ErrorCode.FLOW_CONTROL_ERROR, "flow control window too large") self.cond.notify_all() def apply_window_update(self, frame): try: window_update, = struct.unpack('>I', frame.data) except struct.error: raise ConnectionError(constants.ErrorCode.FRAME_SIZE_ERROR, "WINDOW_UPDATE incorrect size") # strip reserved bit window_update = window_update & 0x7fffffff if window_update == 0: self._raise_error(constants.ErrorCode.PROTOCOL_ERROR, "window update must not be zero") self.adjust(window_update) @gen.coroutine def consume(self, amount): while not self.closed and self.size <= 0: yield self.cond.wait() if self.closed: raise StreamClosedError() if self.size < amount: amount = self.size if self.parent is not None: amount = yield self.parent.consume(amount) self.size -= amount raise gen.Return(amount)
class zip(Stream): """ Combine streams together into a stream of tuples We emit a new tuple once all streams have produce a new tuple. See also -------- combine_latest zip_latest """ _graphviz_orientation = 270 _graphviz_shape = 'triangle' def __init__(self, *upstreams, **kwargs): self.maxsize = kwargs.pop('maxsize', 10) self.buffers = [deque() for _ in upstreams] self.condition = Condition() self.literals = [(i, val) for i, val in enumerate(upstreams) if not isinstance(val, Stream)] self.pack_literals() self.buffers_by_stream = { upstream: buffer for upstream, buffer in builtins.zip(upstreams, self.buffers) if isinstance(upstream, Stream) } upstreams2 = [ upstream for upstream in upstreams if isinstance(upstream, Stream) ] Stream.__init__(self, upstreams=upstreams2, **kwargs) def pack_literals(self): """ Fill buffers for literals whenver we empty them """ for i, val in self.literals: self.buffers[i].append(val) def update(self, x, who=None): L = self.buffers_by_stream[who] # get buffer for stream L.append(x) if len(L) == 1 and all(self.buffers): tup = tuple(buf.popleft() for buf in self.buffers) self.condition.notify_all() if self.literals: self.pack_literals() return self._emit(tup) elif len(L) > self.maxsize: return self.condition.wait()
def __init__(self, *upstreams, **kwargs): self.maxsize = kwargs.pop('maxsize', 10) self.buffers = [deque() for _ in upstreams] self.condition = Condition() self.literals = [(i, val) for i, val in enumerate(upstreams) if not isinstance(val, Stream)] self.pack_literals() self.buffers_by_stream = {upstream: buffer for upstream, buffer in builtins.zip(upstreams, self.buffers) if isinstance(upstream, Stream)} upstreams2 = [upstream for upstream in upstreams if isinstance(upstream, Stream)] Stream.__init__(self, upstreams=upstreams2, **kwargs)
def __init__(self, address, record_dest): """ @brief Construct new instance If record_dest is not empty, create a folder named record_dest and record the received packages there. """ self._address = address self.__record_dest = record_dest if record_dest: if not os.path.isdir(record_dest): os.makedirs(record_dest) self._ioloop = IOLoop.current() self._stop_event = Event() self._is_stopped = Condition() self._socket = None self.__last_package = 0
class CounterCondition(object): def __init__(self): self.condition = Condition() self.counter = 0 def increment(self, value=1): self.counter += value self.condition.notify_all() @gen.coroutine def wait_until(self, value): while True: yield self.condition.wait() if self.counter >= value: self.counter -= value return
def __init__(self, session, home): self.scrapper = Scrapper(session, home) self.session = session self.home = home self.condition = Condition() tornado.ioloop.IOLoop.current()\ .spawn_callback(self.main)
def __init__(self, core_pool_size, queue, reject_handler, coroutine_pool_name=None): self._core_pool_size = core_pool_size self._queue = queue self._reject_handler = reject_handler self._coroutine_pool_name = coroutine_pool_name or \ 'tornado-coroutine-pool-%s' % uuid.uuid1().hex self._core_coroutines_condition = Condition() self._core_coroutines = {} self._core_coroutines_wait_condition = Condition() self._shutting_down = False self._shuted_down = False self._initialize_core_coroutines()
class latest(Stream): """ Drop held-up data and emit the latest result This allows you to skip intermediate elements in the stream if there is some back pressure causing a slowdown. Use this when you only care about the latest elements, and are willing to lose older data. This passes through values without modification otherwise. Examples -------- >>> source.map(f).latest().map(g) # doctest: +SKIP """ _graphviz_shape = 'octagon' def __init__(self, upstream, **kwargs): self.condition = Condition() self.next = [] Stream.__init__(self, upstream, ensure_io_loop=True, **kwargs) self.loop.add_callback(self.cb) def update(self, x, who=None): self.next = [x] self.loop.add_callback(self.condition.notify) @gen.coroutine def cb(self): while True: yield self.condition.wait() [x] = self.next yield self._emit(x)
def __init__ (self, device_server, stream, address): self.fw_version = 0.0 self.recv_msg_cond = Condition() self.recv_msg = {} self.send_msg_sem = Semaphore(1) self.pending_request_cnt = 0 self.device_server = device_server self.stream = stream self.address = address self.stream.set_nodelay(True) self.timeout_handler_onlinecheck = None self.timeout_handler_offline = None self.killed = False self.sn = "" self.private_key = "" self.node_id = 0 self.iv = None self.cipher_down = None self.cipher_up = None #self.state_waiters = [] #self.state_happened = [] self.event_waiters = [] self.event_happened = [] self.ota_ing = False self.ota_notify_done_future = None self.post_ota = False self.online_status = True
class PingHandler(firenado.tornadoweb.TornadoHandler): def __init__(self, application, request, **kwargs): super(PingHandler, self).__init__(application, request, **kwargs) self.callback_queue = None self.condition = Condition() self.response = None self.corr_id = str(uuid.uuid4()) self.in_channel = self.application.get_app_component().rabbitmq[ 'client'].channels['in'] @gen.coroutine def post(self): self.in_channel.queue_declare(exclusive=True, callback=self.on_request_queue_declared) yield self.condition.wait() self.write(self.response) def on_request_queue_declared(self, response): logger.info('Request temporary queue declared.') self.callback_queue = response.method.queue self.in_channel.basic_consume(self.on_response, no_ack=True, queue=self.callback_queue) self.in_channel.basic_publish( exchange='', routing_key='ping_rpc_queue', properties=pika.BasicProperties( reply_to=self.callback_queue, correlation_id=self.corr_id, ), body=self.request.body) def on_response(self, ch, method, props, body): if self.corr_id == props.correlation_id: self.response = { 'data': body.decode("utf-8"), 'date': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), } self.in_channel.queue_delete(queue=self.callback_queue) self.condition.notify()
def test_future_close_callback(self): # Regression test for interaction between the Future read interfaces # and IOStream._maybe_add_error_listener. rs, ws = yield self.make_iostream_pair() closed = [False] cond = Condition() def close_callback(): closed[0] = True cond.notify() rs.set_close_callback(close_callback) try: ws.write(b'a') res = yield rs.read_bytes(1) self.assertEqual(res, b'a') self.assertFalse(closed[0]) ws.close() yield cond.wait() self.assertTrue(closed[0]) finally: rs.close() ws.close()
class GameScrapperWorker(object): def __init__(self, session, home): self.scrapper = Scrapper(session, home) self.session = session self.home = home self.condition = Condition() tornado.ioloop.IOLoop.current()\ .spawn_callback(self.main) @tornado.gen.coroutine def main(self): _log().info('scrapper sleeping') yield self.condition.wait() _log().info('scrapper woke up') self.scrapper.scrap_missing() tornado.ioloop.IOLoop.current()\ .spawn_callback(self.main) raise tornado.gen.Return(False)
def __init__(self, tchannel, score_threshold=None): """Initializes a new PeerGroup. :param tchannel: TChannel used for communication by this PeerGroup :param score_threshold: A value in the ``[0, 1]`` range. If specifiede, this requires that chosen peers havea score higher than this value when performing requests. """ self.tchannel = tchannel self._score_threshold = score_threshold # Dictionary from hostport to Peer. self._peers = {} # Notified when a reset is performed. This allows multiple coroutines # to block on the same reset. self._resetting = False self._reset_condition = Condition()
class ImporterWorker(object): def __init__(self, session, home, scrapper): self.session = session self.home = home self.condition = Condition() self.scrapper = scrapper tornado.ioloop.IOLoop.current()\ .spawn_callback(self.main) @tornado.gen.coroutine def main(self): _log().info('importer sleeping') yield self.condition.wait() _log().info('importer woke up') changed = True while changed: changed = yield self.work() tornado.ioloop.IOLoop.current()\ .spawn_callback(self.main) @tornado.gen.coroutine def work(self): changed = False fi = slickbird.FileImporter(self.session, self.home) for f in self.session.query(orm.Importerfile)\ .filter(orm.Importerfile.status == 'scanning'): changed = True r, status = fi.file_import(f.filename) f.status = status if status == 'moved': self.scrapper.condition.notify() yield tornado.gen.moment self.session.commit() self.scrapper.condition.notify() raise tornado.gen.Return(changed)
class InMemStream(Stream): def __init__(self, buf=None, auto_close=True): """In-Memory based stream :param buf: the buffer for the in memory stream """ self._stream = deque() if buf: self._stream.append(buf) self.state = StreamState.init self._condition = Condition() self.auto_close = auto_close self.exception = None def clone(self): new_stream = InMemStream() new_stream.state = self.state new_stream.auto_close = self.auto_close new_stream._stream = deque(self._stream) return new_stream def read(self): def read_chunk(future): if self.exception: future.set_exception(self.exception) return future chunk = "" while len(self._stream) and len(chunk) < common.MAX_PAYLOAD_SIZE: chunk += self._stream.popleft() future.set_result(chunk) return future read_future = tornado.concurrent.Future() # We're not ready yet if self.state != StreamState.completed and not len(self._stream): wait_future = self._condition.wait() wait_future.add_done_callback( lambda f: f.exception() or read_chunk(read_future) ) return read_future return read_chunk(read_future) def write(self, chunk): if self.exception: raise self.exception if self.state == StreamState.completed: raise UnexpectedError("Stream has been closed.") if chunk: self._stream.append(chunk) self._condition.notify() # This needs to return a future to match the async interface. r = tornado.concurrent.Future() r.set_result(None) return r def set_exception(self, exception): self.exception = exception self.close() def close(self): self.state = StreamState.completed self._condition.notify()
def __init__(self): # Image data self._frame = None # Flow control self._condition = Condition()
def __init__(self, parent, stream_id, initial_window_size): self.parent = parent self.stream_id = stream_id self.cond = Condition() self.closed = False self.size = initial_window_size
class DeviceConnection(object): state_waiters = {} state_happened = {} def __init__ (self, device_server, stream, address): self.fw_version = 0.0 self.recv_msg_cond = Condition() self.recv_msg = {} self.send_msg_sem = Semaphore(1) self.pending_request_cnt = 0 self.device_server = device_server self.stream = stream self.address = address self.stream.set_nodelay(True) self.timeout_handler_onlinecheck = None self.timeout_handler_offline = None self.killed = False self.sn = "" self.private_key = "" self.node_id = 0 self.iv = None self.cipher_down = None self.cipher_up = None #self.state_waiters = [] #self.state_happened = [] self.event_waiters = [] self.event_happened = [] self.ota_ing = False self.ota_notify_done_future = None self.post_ota = False self.online_status = True @gen.coroutine def secure_write (self, data): if self.cipher_down: cipher_text = self.cipher_down.encrypt(pad(data)) yield self.stream.write(cipher_text) @gen.coroutine def wait_hello (self): try: self._wait_hello_future = self.stream.read_bytes(64) #read 64bytes: 32bytes SN + 32bytes signature signed with private key str1 = yield gen.with_timeout(timedelta(seconds=10), self._wait_hello_future, io_loop=ioloop.IOLoop.current()) self.idle_time = 0 #reset the idle time counter if len(str1) != 64: self.stream.write("sorry\r\n") yield gen.sleep(0.1) self.kill_myself() gen_log.debug("receive length != 64") raise gen.Return(100) # length not match 64 if re.match(r'@\d\.\d', str1[0:4]): #new version firmware self._wait_hello_future = self.stream.read_bytes(4) #read another 4bytes str2 = yield gen.with_timeout(timedelta(seconds=10), self._wait_hello_future, io_loop=ioloop.IOLoop.current()) self.idle_time = 0 #reset the idle time counter if len(str2) != 4: self.stream.write("sorry\r\n") yield gen.sleep(0.1) self.kill_myself() gen_log.debug("receive length != 68") raise gen.Return(100) # length not match 64 str1 += str2 self.fw_version = float(str1[1:4]) sn = str1[4:36] sig = str1[36:68] else: #for version < 1.1 sn = str1[0:32] sig = str1[32:64] gen_log.info("accepted sn: %s @fw_version %.1f" % (sn, self.fw_version)) #query the sn from database node = None cur = self.device_server.cur cur.execute('select * from nodes where node_sn="%s"'%sn) rows = cur.fetchall() if len(rows) > 0: node = rows[0] if not node: self.stream.write("sorry\r\n") yield gen.sleep(0.1) self.kill_myself() gen_log.info("node sn not found") raise gen.Return(101) #node not found key = node['private_key'] key = key.encode("ascii") sig0 = hmac.new(key, msg=sn, digestmod=hashlib.sha256).digest() gen_log.debug("sig: "+ binascii.hexlify(sig)) gen_log.debug("sig calc:"+ binascii.hexlify(sig0)) if sig0 == sig: #send IV + AES Key self.sn = sn self.private_key = key self.node_id = str(node['node_id']) gen_log.info("valid hello packet from node %s" % self.node_id) #remove the junk connection of the same sn ioloop.IOLoop.current().add_callback(self.device_server.remove_junk_connection, self) #init aes self.iv = Random.new().read(AES.block_size) self.cipher_down = AES.new(key, AES.MODE_CFB, self.iv, segment_size=128) if self.fw_version > 1.0: self.cipher_up = AES.new(key, AES.MODE_CFB, self.iv, segment_size=128) else: #for old version self.cipher_up = self.cipher_down cipher_text = self.iv + self.cipher_down.encrypt(pad("hello")) gen_log.debug("cipher text: "+ cipher_text.encode('hex')) self.stream.write(cipher_text) raise gen.Return(0) else: self.stream.write("sorry\r\n") yield gen.sleep(0.1) self.kill_myself() gen_log.error("signature not match: %s %s" % (sig, sig0)) raise gen.Return(102) #sig not match except gen.TimeoutError: self.kill_myself() raise gen.Return(1) except iostream.StreamClosedError: self.kill_myself() raise gen.Return(2) #ioloop.IOLoop.current().add_future(self._serving_future, lambda future: future.result()) @gen.coroutine def _loop_reading_input (self): line = "" piece = "" while not self.killed: msg = "" try: msg = yield self.stream.read_bytes(16) msg = unpad(self.cipher_up.decrypt(msg)) line += msg while line.find('\r\n') > -1: #reset the timeout if self.timeout_handler_onlinecheck: ioloop.IOLoop.current().remove_timeout(self.timeout_handler_onlinecheck) self.timeout_handler_onlinecheck = ioloop.IOLoop.current().call_later(60, self._online_check) if self.timeout_handler_offline: ioloop.IOLoop.current().remove_timeout(self.timeout_handler_offline) self.timeout_handler_offline = ioloop.IOLoop.current().call_later(70, self._callback_when_offline) index = line.find('\r\n') piece = line[:index+2] line = line[index+2:] piece = piece.strip("\r\n") if piece in ['##ALIVE##']: gen_log.info('Node %s alive on %s channel!' % (self.node_id, self.device_server.role)) continue json_obj = json.loads(piece) gen_log.info('Node %s recv json on %s channel' % (self.node_id, self.device_server.role)) gen_log.debug('%s' % str(json_obj)) try: state = None event = None if json_obj['msg_type'] == 'online_status': if json_obj['msg'] in ['1',1,True]: self.online_status = True else: self.online_status = False continue elif json_obj['msg_type'] == 'ota_trig_ack': state = ('going', 'Node has been notified...') self.ota_ing = True if self.ota_notify_done_future: self.ota_notify_done_future.set_result(1) elif json_obj['msg_type'] == 'ota_status': if json_obj['msg'] == 'started': state = ('going', 'Downloading the firmware...') else: state = ('error', 'Failed to start the downloading.') self.post_ota = True elif json_obj['msg_type'] == 'ota_result': if json_obj['msg'] == 'success': state = ('done', 'Firmware updated.') else: state = ('error', 'Update failed. Please reboot the node and retry.') self.post_ota = True self.ota_ing = False elif json_obj['msg_type'] == 'event': event = json_obj event.pop('msg_type') gen_log.debug("state: ") gen_log.debug(state) gen_log.debug("event: ") gen_log.debug(event) if state: #print self.state_waiters #print self.state_happened if self.state_waiters and self.sn in self.state_waiters and len(self.state_waiters[self.sn]) > 0: f = self.state_waiters[self.sn].pop(0) f.set_result(state) if len(self.state_waiters[self.sn]) == 0: del self.state_waiters[self.sn] elif self.state_happened and self.sn in self.state_happened: self.state_happened[self.sn].append(state) else: self.state_happened[self.sn] = [state] elif event: if len(self.event_waiters) == 0: self.event_happened.append(event) else: for future in self.event_waiters: future.set_result(event) self.event_waiters = [] else: self.recv_msg = json_obj self.recv_msg_cond.notify() yield gen.moment except Exception,e: gen_log.warn("Node %s: %s" % (self.node_id ,str(e))) except iostream.StreamClosedError: gen_log.error("StreamClosedError when reading from node %s" % self.node_id) self.kill_myself() return except ValueError: gen_log.warn("Node %s: %s can not be decoded into json" % (self.node_id, piece)) except Exception,e: gen_log.error("Node %s: %s" % (self.node_id ,str(e))) self.kill_myself() return yield gen.moment
class DeviceConnection(object): state_waiters = {} state_happened = {} def __init__ (self, device_server, stream, address): self.recv_msg_cond = Condition() self.recv_msg = {} self.send_msg_sem = Semaphore(1) self.pending_request_cnt = 0 self.device_server = device_server self.stream = stream self.address = address self.stream.set_nodelay(True) self.idle_time = 0; self.killed = False self.sn = "" self.private_key = "" self.node_id = 0 self.name = "" self.iv = None self.cipher = None #self.state_waiters = [] #self.state_happened = [] self.event_waiters = [] self.event_happened = [] self.ota_ing = False self.ota_notify_done_future = None self.post_ota = False self.online_status = True @gen.coroutine def secure_write (self, data): if self.cipher: cipher_text = self.cipher.encrypt(pad(data)) yield self.stream.write(cipher_text) @gen.coroutine def wait_hello (self): try: self._wait_hello_future = self.stream.read_bytes(64) #read 64bytes: 32bytes SN + 32bytes signature signed with private key str = yield gen.with_timeout(timedelta(seconds=10), self._wait_hello_future, io_loop=self.stream.io_loop) self.idle_time = 0 #reset the idle time counter if len(str) != 64: self.stream.write("sorry\r\n") yield gen.sleep(0.1) self.kill_myself() gen_log.debug("receive length != 64") raise gen.Return(100) # length not match 64 sn = str[0:32] sig = str[32:64] gen_log.info("accepted sn: "+ sn) #query the sn from database node = None for n in NODES_DATABASE: if n['node_sn'] == sn: node = n break if not node: self.stream.write("sorry\r\n") yield gen.sleep(0.1) self.kill_myself() gen_log.info("node sn not found") raise gen.Return(101) #node not found key = node['node_key'] key = key.encode("ascii") sig0 = hmac.new(key, msg=sn, digestmod=hashlib.sha256).digest() gen_log.debug("sig: "+ binascii.hexlify(sig)) gen_log.debug("sig calc:"+ binascii.hexlify(sig0)) if sig0 == sig: #send IV + AES Key self.sn = sn self.private_key = key self.node_id = node['node_sn'] self.name = node['name'] gen_log.info("valid hello packet from node %s" % self.name) #remove the junk connection of the same sn self.stream.io_loop.add_callback(self.device_server.remove_junk_connection, self) #init aes self.iv = Random.new().read(AES.block_size) self.cipher = AES.new(key, AES.MODE_CFB, self.iv, segment_size=128) cipher_text = self.iv + self.cipher.encrypt(pad("hello")) gen_log.debug("cipher text: "+ cipher_text.encode('hex')) self.stream.write(cipher_text) raise gen.Return(0) else: self.stream.write("sorry\r\n") yield gen.sleep(0.1) self.kill_myself() gen_log.error("signature not match: %s %s" % (sig, sig0)) raise gen.Return(102) #sig not match except gen.TimeoutError: self.kill_myself() raise gen.Return(1) except iostream.StreamClosedError: self.kill_myself() raise gen.Return(2) #self.stream.io_loop.add_future(self._serving_future, lambda future: future.result()) @gen.coroutine def _loop_reading_input (self): line = "" piece = "" while not self.killed: msg = "" try: msg = yield self.stream.read_bytes(16) msg = unpad(self.cipher.decrypt(msg)) self.idle_time = 0 #reset the idle time counter line += msg while line.find('\r\n') > -1: index = line.find('\r\n') piece = line[:index+2] line = line[index+2:] piece = piece.strip("\r\n") json_obj = json.loads(piece) gen_log.info('Node %s: recv json: %s' % (self.name, str(json_obj))) try: state = None event = None if json_obj['msg_type'] == 'online_status': if json_obj['msg'] in ['1',1,True]: self.online_status = True else: self.online_status = False continue elif json_obj['msg_type'] == 'ota_trig_ack': state = ('going', 'Node has been notified...') self.ota_ing = True if self.ota_notify_done_future: self.ota_notify_done_future.set_result(1) elif json_obj['msg_type'] == 'ota_status': if json_obj['msg'] == 'started': state = ('going', 'Downloading the firmware...') else: state = ('error', 'Failed to start the downloading.') self.post_ota = True elif json_obj['msg_type'] == 'ota_result': if json_obj['msg'] == 'success': state = ('done', 'Firmware updated.') else: state = ('error', 'Update failed. Please reboot the node and retry.') self.post_ota = True self.ota_ing = False elif json_obj['msg_type'] == 'event': event = json_obj event.pop('msg_type') gen_log.debug(state) gen_log.debug(event) if state: #print self.state_waiters #print self.state_happened if self.state_waiters and self.sn in self.state_waiters and len(self.state_waiters[self.sn]) > 0: f = self.state_waiters[self.sn].pop(0) f.set_result(state) if len(self.state_waiters[self.sn]) == 0: del self.state_waiters[self.sn] elif self.state_happened and self.sn in self.state_happened: self.state_happened[self.sn].append(state) else: self.state_happened[self.sn] = [state] elif event: if len(self.event_waiters) == 0: self.event_happened.append(event) else: for future in self.event_waiters: future.set_result(event) self.event_waiters = [] else: self.recv_msg = json_obj self.recv_msg_cond.notify() yield gen.moment except Exception,e: gen_log.warn("Node %s: %s" % (self.name ,str(e))) except iostream.StreamClosedError: gen_log.error("StreamClosedError when reading from node %s" % self.name) self.kill_myself() return except ValueError: gen_log.warn("Node %s: %s can not be decoded into json" % (self.name, piece)) except Exception,e: gen_log.error("Node %s: %s" % (self.name ,str(e))) self.kill_myself() return yield gen.moment
def get_data(cls, account, source_filter, limit=100, skip=0): source_filter = OneDriveFileFilter(source_filter) if source_filter.file is None: raise ValueError("required parameter file missing") app_log.info("Starting to retrieve file for {}".format(account._id)) client = AsyncHTTPClient() uri = "https://api.onedrive.com/v1.0/drive/items/{}/content".format(source_filter.file) lock = Condition() def crawl_url(url): # some yummy regex location_header_regex = re.compile(r"^Location:\s?(?P<uri>http:/{2}\S+)") http_status_regex = re.compile(r"^HTTP/[\d\.]+\s(?P<status>\d+)") receiving_file = False # define our callbacks def header_callback(header): m = http_status_regex.match(header) if m is not None: # process our HTTP status header status = m.group("status") if int(status) == 200: # if we're 200, we're receiving the file, not just a redirect app_log.info("Receiving file {} for account {}".format(source_filter.file, account._id)) global receiving_file receiving_file = True m = location_header_regex.match(header) if m is not None: # process our location header uri = m.group("uri") # and grab _that_ url app_log.info("Following redirect for file {}".format(source_filter.file)) crawl_url(uri) def stream_callback(chunk): # only dump out chunks that are of the file we're looking for global receiving_file if receiving_file: app_log.info("Writing chunk of {}B".format(chunk.__len__())) cls.write(chunk) def on_completed(resp): if 200 <= resp.code <= 299: lock.notify() oauth_client = account.get_client() uri, headers, body = oauth_client.add_token(url) req = HTTPRequest( uri, headers=headers, body=body, header_callback=header_callback, streaming_callback=stream_callback ) client.fetch(req, callback=on_completed) crawl_url(uri) # wait for us to complete try: yield lock.wait(timeout=timedelta(seconds=MAXIMUM_REQ_TIME)) app_log.info("File {} retrieved successfully".format(source_filter.file)) except gen.TimeoutError: app_log.error("Request for file {} => {} timed out!".format(source_filter.file, account._id))
class PeerGroup(object): """A PeerGroup represents a collection of Peers. Requests routed through a PeerGroup can be sent to either a specific peer or a peer chosen at random. """ def __init__(self, tchannel, score_threshold=None): """Initializes a new PeerGroup. :param tchannel: TChannel used for communication by this PeerGroup :param score_threshold: A value in the ``[0, 1]`` range. If specifiede, this requires that chosen peers havea score higher than this value when performing requests. """ self.tchannel = tchannel self._score_threshold = score_threshold # Dictionary from hostport to Peer. self._peers = {} # Notified when a reset is performed. This allows multiple coroutines # to block on the same reset. self._resetting = False # We'll create a Condition here later. We want to avoid it right now # because it has a side-effect of scheduling some dummy work on the # ioloop, which prevents us from forking (if you're into that). self._reset_condition = None def __str__(self): return "<PeerGroup peers=%s>" % str(self._peers) @gen.coroutine def clear(self): """Reset this PeerGroup. This closes all connections to all known peers and forgets about these peers. :returns: A Future that resolves with a value of None when the operation has finished """ if self._resetting: # If someone else is already resetting the PeerGroup, just block # on them to be finished. yield self._reset_condition.wait() raise gen.Return(None) self._resetting = True if self._reset_condition is None: self._reset_condition = Condition() try: for peer in self._peers.values(): peer.close() finally: self._peers = {} self._resetting = False self._reset_condition.notify_all() def get(self, hostport): """Get a Peer for the given destination. A new Peer is added and returned if one does not already exist for the given host-port. Otherwise, the existing Peer is returned. """ assert hostport, "hostport is required" if hostport not in self._peers: self._peers[hostport] = Peer(self.tchannel, hostport) return self._peers[hostport] def lookup(self, hostport): """Look up a Peer for the given host and port. Returns None if a Peer for the given host-port does not exist. """ assert hostport, "hostport is required" return self._peers.get(hostport, None) def remove(self, hostport): """Delete the Peer for the given host port. Does nothing if a matching Peer does not exist. :returns: The removed Peer """ assert hostport, "hostport is required" return self._peers.pop(hostport, None) def add(self, peer): """Add an existing Peer to this group. A peer for the given host-port must not already exist in the group. """ assert peer, "peer is required" if isinstance(peer, basestring): # Assume strings are host-ports peer = Peer(self.tchannel, peer) assert peer.hostport not in self._peers, "%s already has a peer" % peer.hostport self._peers[peer.hostport] = peer @property def hosts(self): """Get all host-ports managed by this PeerGroup.""" return self._peers.keys() @property def peers(self): """Get all Peers managed by this PeerGroup.""" return self._peers.values() def request(self, service, hostport=None, **kwargs): """Initiate a new request through this PeerGroup. :param hostport: If specified, requests will be sent to the specific host. Otherwise, a known peer will be picked at random. :param service: Name of the service being called. Defaults to an empty string. :param service_threshold: If ``hostport`` was not specified, this specifies the score threshold at or below which peers will be ignored. """ return PeerClientOperation(peer_group=self, service=service, hostport=hostport, **kwargs) def choose(self, hostport=None, score_threshold=None, blacklist=None): """Choose a Peer that matches the given criteria. The Peer with the highest score will be chosen. :param hostport: Specifies that the returned Peer must be for the given host-port. Without this, all peers managed by this PeerGroup are candidates. If this is present, ``score_threshold`` is ignored. :param score_threshold: If specified, Peers with a score equal to or below this will be ignored. Defaults to the value specified when the PeerGroup was initialized. :param blacklist: Peers on the blacklist won't be chosen. :returns: A Peer that matches all the requested criteria or None if no such Peer was found. """ blacklist = blacklist or set() if hostport: return self.get(hostport) score_threshold = score_threshold or self._score_threshold or 0 chosen_peer = None chosen_score = 0 hosts = self._peers.viewkeys() - blacklist for host in hosts: peer = self.get(host) score = peer.state.score() if score <= score_threshold: continue if score > chosen_score: chosen_peer = peer chosen_score = score return chosen_peer