def on_server_loaded(server_context): messages['workers'] = {'interval': 1000, 'deque': deque(maxlen=1000), 'times': deque(maxlen=1000), 'condition': Condition()} server_context.add_periodic_callback(lambda: http_get('workers'), 1000) messages['tasks'] = {'interval': 100, 'deque': deque(maxlen=1000), 'times': deque(maxlen=1000), 'condition': Condition()} server_context.add_periodic_callback(lambda: http_get('tasks'), 100)
def __init__(self, child, loop=None): self.condition = Condition() self.next = [] Stream.__init__(self, child, loop=loop) self.loop.add_callback(self.cb)
def __init__ (self, device_server, stream, address): self.recv_msg_cond = Condition() self.recv_msg = {} self.send_msg_sem = Semaphore(1) self.pending_request_cnt = 0 self.device_server = device_server self.stream = stream self.address = address self.stream.set_nodelay(True) self.idle_time = 0; self.killed = False self.sn = "" self.private_key = "" self.node_id = 0 self.name = "" self.iv = None self.cipher = None #self.state_waiters = [] #self.state_happened = [] self.event_waiters = [] self.event_happened = [] self.ota_ing = False self.ota_notify_done_future = None self.post_ota = False self.online_status = True
def __init__(self, upstream, **kwargs): self.condition = Condition() self.next = [] Stream.__init__(self, upstream, ensure_io_loop=True, **kwargs) self.loop.add_callback(self.cb)
def prepare(self): self.id = str(uuid.uuid4()) self.final_hyp = "" self.worker_done = Condition() self.user_id = self.request.headers.get("device-id", "none") self.content_id = self.request.headers.get("content-id", "none") logging.info("%s: OPEN: user='******', content='%s'" % (self.id, self.user_id, self.content_id)) self.worker = None self.error_status = 0 self.error_message = None #Waiter thread for final hypothesis: # self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) try: self.worker = self.application.available_workers.pop() self.application.send_status_update() logging.info("%s: Using worker %s" % (self.id, self.__str__())) self.worker.set_client_socket(self) content_type = self.request.headers.get("Content-Type", None) if content_type: content_type = content_type_to_caps(content_type) logging.info("%s: Using content type: %s" % (self.id, content_type)) self.worker.write_message(json.dumps(dict(id=self.id, content_type=content_type, user_id=self.user_id, content_id=self.content_id))) except KeyError: logging.warn("%s: No worker available for client request" % self.id) self.set_status(503) self.finish("No workers available")
def __init__ (self, device_server, stream, address, conn_pool): self.fw_version = 0.0 self.recv_msg_cond = Condition() self.recv_msg = {} self.send_msg_sem = Semaphore(1) self.pending_request_cnt = 0 self.device_server = device_server self.device_server_conn_pool = conn_pool self.stream = stream self.address = address self.stream.set_nodelay(True) self.stream.set_close_callback(self.on_close) self.timeout_handler_onlinecheck = None self.timeout_handler_offline = None self.killed = False self.is_junk = False self.sn = "" self.private_key = "" self.node_id = "" self.user_id = "" self.iv = None self.cipher_down = None self.cipher_up = None self.event_waiters = [] self.event_happened = [] self.ota_ing = False self.ota_notify_done_future = None self.post_ota = False self.online_status = True
async def refresh(self, fetch_packages=True): # TODO: Use python-apt python lib rather than command line for updates if self.refresh_condition is None: self.refresh_condition = Condition() else: self.refresh_condition.wait() return try: if fetch_packages: await self.execute_cmd(f"{APT_CMD} update", timeout=300., retries=3) res = await self.execute_cmd_with_response("apt list --upgradable", timeout=60.) pkg_list = [p.strip() for p in res.split("\n") if p.strip()] if pkg_list: pkg_list = pkg_list[2:] self.available_packages = [ p.split("/", maxsplit=1)[0] for p in pkg_list ] pkg_list = "\n".join(self.available_packages) logging.info( f"Detected {len(self.available_packages)} package updates:" f"\n{pkg_list}") except Exception: logging.exception("Error Refreshing System Packages") self.init_evt.set() self.refresh_condition.notify_all() self.refresh_condition = None
def clear(self): """Reset this PeerGroup. This closes all connections to all known peers and forgets about these peers. :returns: A Future that resolves with a value of None when the operation has finished """ if self._resetting: # If someone else is already resetting the PeerGroup, just block # on them to be finished. yield self._reset_condition.wait() raise gen.Return(None) self._resetting = True if self._reset_condition is None: self._reset_condition = Condition() try: for peer in self._peers.values(): peer.close() finally: self._peers = {} self._resetting = False self._reset_condition.notify_all()
def __init__(self, core_pool_size, queue, reject_handler, coroutine_pool_name=None): self._core_pool_size = core_pool_size self._queue = queue self._reject_handler = reject_handler self._coroutine_pool_name = coroutine_pool_name or \ 'tornado-coroutine-pool-%s' % uuid.uuid1().hex self._core_coroutines_condition = Condition() self._core_coroutines = {} self._core_coroutines_wait_condition = Condition() self._shutting_down = False self._shuted_down = False self._initialize_core_coroutines()
def get_data(cls, account, source_filter, limit=100, skip=0): """ Gathers card information from Google Sheets GET https://spreadsheets.google.com/feeds/list/[spreadsheet]/[worksheet]/private/full """ if not account or not account.enabled: raise ValueError('cannot gather information without an account') client = AsyncHTTPClient() if source_filter.spreadsheet is None: raise ValueError('required parameter spreadsheet missing') if source_filter.worksheet is None: raise ValueError('required parameter worksheet missing') uri = "https://docs.google.com/spreadsheets/d/{}/export?format=csv&gid={}".format( source_filter.spreadsheet, source_filter.worksheet) app_log.info("Start retrieval of worksheet {}/{} for {}".format( source_filter.spreadsheet, source_filter.worksheet, account._id)) lock = Condition() oauth_client = account.get_client() uri, headers, body = oauth_client.add_token(uri) req = HTTPRequest(uri, headers=headers, body=body, streaming_callback=lambda c: cls.write(c)) client.fetch(req, callback=lambda r: lock.notify()) yield lock.wait(timeout=timedelta(seconds=MAXIMUM_REQ_TIME)) app_log.info("Finished retrieving worksheet for {}".format( account._id))
def __init__(self, upstream, loop=None): loop = loop or upstream.loop or IOLoop.current() self.condition = Condition() self.next = [] Stream.__init__(self, upstream, loop=loop) self.loop.add_callback(self.cb)
def __init__(self, application, request, **kwargs): super(PingHandler, self).__init__(application, request, **kwargs) self.callback_queue = None self.condition = Condition() self.response = None self.corr_id = str(uuid.uuid4()) self.in_channel = self.application.get_app_component().rabbitmq[ 'client'].channels['in']
def __init__(self, child, limit=10, client=None): self.client = client or default_client() self.queue = Queue(maxsize=limit) self.condition = Condition() Stream.__init__(self, child) self.client.loop.add_callback(self.cb)
def __init__(self, address): """ @brief Construct new instance """ self._address = address self._ioloop = IOLoop.current() self._stop_event = Event() self._is_stopped = Condition() self._socket = None
def send_req(self, req): req["uuid"] = str(uuid.uuid4()) self.send_msg(json.dumps(req, default=json_util.default, indent=4)) self.reqLocks[req["uuid"]] = Condition() yield self.reqLocks[req["uuid"]].wait() res = self.resList[req["uuid"]] # delete record self.reqLocks.pop(req["uuid"], None) self.resList.pop(req["uuid"], None) raise gen.Return(res)
async def refresh(self): if self.refresh_condition is None: self.refresh_condition = Condition() else: self.refresh_condition.wait() return try: await self._check_version() except Exception: logging.exception("Error Refreshing git state") self.init_evt.set() self.refresh_condition.notify_all() self.refresh_condition = None
def __init__(self, buf=None, auto_close=True): """In-Memory based stream :param buf: the buffer for the in memory stream """ self._stream = deque() if buf: self._stream.append(buf) self.state = StreamState.init self._condition = Condition() self.auto_close = auto_close self.exception = None
def __init__(self, *upstreams, **kwargs): self.maxsize = kwargs.pop('maxsize', 10) self.condition = Condition() self.literals = [(i, val) for i, val in enumerate(upstreams) if not isinstance(val, Stream)] self.buffers = {upstream: deque() for upstream in upstreams if isinstance(upstream, Stream)} upstreams2 = [upstream for upstream in upstreams if isinstance(upstream, Stream)] Stream.__init__(self, upstreams=upstreams2, **kwargs)
async def refresh(self): if self.refresh_condition is None: self.refresh_condition = Condition() else: self.refresh_condition.wait() return try: self._get_local_version() await self._get_remote_version() except Exception: logging.exception("Error Refreshing Client") self.init_evt.set() self.refresh_condition.notify_all() self.refresh_condition = None
def prepare(self): self.worker = None self.filePath = None self.uuid = str(uuid.uuid4()) self.set_status(200, "Initial statut") self.waitResponse = Condition() self.waitWorker = Condition() if self.request.method != 'POST': logging.debug("Received a non-POST request") self.set_status( 403, "Wrong request, server handles only POST requests") self.finish() #File Retrieval # TODO: Adapt input to existing controller API if 'wavFile' not in self.request.files.keys(): self.set_status( 403, "POST request must contain a 'file_to_transcript' field.") self.finish() logging.debug( "POST request from %s does not contain 'file_to_transcript' field." ) temp_file = self.request.files['wavFile'][0]['body'] self.temp_file = temp_file #Writing file try: f = open(TEMP_FILE_PATH + self.uuid + '.wav', 'wb') except IOError: logging.error("Could not write file.") self.set_status( 500, "Server error: Counldn't write file on server side.") self.finish() else: f.write(temp_file) self.filePath = TEMP_FILE_PATH + self.uuid + '.wav' logging.debug("File correctly received from client")
def __init__(self, *upstreams, **kwargs): self.maxsize = kwargs.pop('maxsize', 10) self.buffers = [deque() for _ in upstreams] self.condition = Condition() self.literals = [(i, val) for i, val in enumerate(upstreams) if not isinstance(val, Stream)] self.pack_literals() self.buffers_by_stream = {upstream: buffer for upstream, buffer in builtins.zip(upstreams, self.buffers) if isinstance(upstream, Stream)} upstreams2 = [upstream for upstream in upstreams if isinstance(upstream, Stream)] Stream.__init__(self, upstreams=upstreams2, **kwargs)
def __init__(self, address, record_dest): """ @brief Construct new instance If record_dest is not empty, create a folder named record_dest and record the received packages there. """ self._address = address self.__record_dest = record_dest if record_dest: if not os.path.isdir(record_dest): os.makedirs(record_dest) self._ioloop = IOLoop.current() self._stop_event = Event() self._is_stopped = Condition() self._socket = None self.__last_package = 0
def __init__(self, *children, **kwargs): self.maxsize = kwargs.pop('maxsize', 10) self.buffers = [deque() for _ in children] self.condition = Condition() self.literals = [(i, val) for i, val in enumerate(children) if not isinstance(val, Stream)] self.pack_literals() self.buffers_by_stream = { child: buffer for child, buffer in builtins.zip(children, self.buffers) if isinstance(child, Stream) } children2 = [child for child in children if isinstance(child, Stream)] Stream.__init__(self, children=children2, **kwargs)
def test_future_close_callback(self): # Regression test for interaction between the Future read interfaces # and IOStream._maybe_add_error_listener. rs, ws = yield self.make_iostream_pair() closed = [False] cond = Condition() def close_callback(): closed[0] = True cond.notify() rs.set_close_callback(close_callback) try: ws.write(b"a") res = yield rs.read_bytes(1) self.assertEqual(res, b"a") self.assertFalse(closed[0]) ws.close() yield cond.wait() self.assertTrue(closed[0]) finally: rs.close() ws.close()
def get_data(cls, account, source_filter, limit=100, skip=0): source_filter = OneDriveFileFilter(source_filter) if source_filter.file is None: raise ValueError('required parameter file missing') app_log.info("Starting to retrieve file for {}".format(account._id)) client = AsyncHTTPClient() uri = "https://api.onedrive.com/v1.0/drive/items/{}/content".format( source_filter.file) lock = Condition() def crawl_url(url): # some yummy regex location_header_regex = re.compile( r'^Location:\s?(?P<uri>http:/{2}\S+)') http_status_regex = re.compile(r'^HTTP/[\d\.]+\s(?P<status>\d+)') receiving_file = False # define our callbacks def header_callback(header): m = http_status_regex.match(header) if m is not None: # process our HTTP status header status = m.group('status') if int(status) == 200: # if we're 200, we're receiving the file, not just a redirect app_log.info("Receiving file {} for account {}".format( source_filter.file, account._id)) global receiving_file receiving_file = True m = location_header_regex.match(header) if m is not None: # process our location header uri = m.group('uri') # and grab _that_ url app_log.info("Following redirect for file {}".format( source_filter.file)) crawl_url(uri) def stream_callback(chunk): # only dump out chunks that are of the file we're looking for global receiving_file if receiving_file: app_log.info("Writing chunk of {}B".format( chunk.__len__())) cls.write(chunk) def on_completed(resp): if 200 <= resp.code <= 299: lock.notify() oauth_client = account.get_client() uri, headers, body = oauth_client.add_token(url) req = HTTPRequest(uri, headers=headers, body=body, header_callback=header_callback, streaming_callback=stream_callback) client.fetch(req, callback=on_completed) crawl_url(uri) # wait for us to complete try: yield lock.wait(timeout=timedelta(seconds=MAXIMUM_REQ_TIME)) app_log.info("File {} retrieved successfully".format( source_filter.file)) except gen.TimeoutError: app_log.error("Request for file {} => {} timed out!".format( source_filter.file, account._id))
def __init__(self): self.cond = Condition() self.cache = [] # all messages added here self.cache_size = 200 # holds 200 newest msgs
async def acquire_url_lock(self, url): if url not in BaseHandler.url_locks: BaseHandler.url_locks[url] = Condition() else: return await BaseHandler.url_locks[url].wait()
def acquire_url_lock(self, url): if url not in BaseHandler.url_locks: BaseHandler.url_locks[url] = Condition() else: yield BaseHandler.url_locks[url].wait()
async def initialize(self, need_fetch=True): if self.init_condition is not None: # No need to initialize multiple requests await self.init_condition.wait() return self.init_condition = Condition() try: await self.update_repo_status() self._verify_repo() if not self.head_detached: # lookup remote via git config self.git_remote = await self.get_config_item( f"branch.{self.git_branch}.remote") # Populate list of current branches blist = await self.list_branches() self.branches = [] for branch in blist: branch = branch.strip() if branch[0] == "*": branch = branch[2:] if branch[0] == "(": continue self.branches.append(branch) if need_fetch: await self.fetch() self.upstream_url = await self.remote("get-url") self.current_commit = await self.rev_parse("HEAD") self.upstream_commit = await self.rev_parse( f"{self.git_remote}/{self.git_branch}") current_version = await self.describe( "--always --tags --long --dirty") upstream_version = await self.describe( f"{self.git_remote}/{self.git_branch} " "--always --tags --long") # Parse GitHub Owner from URL owner_match = re.match(r"https?://[^/]+/([^/]+)", self.upstream_url) self.git_owner = "?" if owner_match is not None: self.git_owner = owner_match.group(1) self.dirty = current_version.endswith("dirty") # Parse Version Info versions = [] for ver in [current_version, upstream_version]: tag_version = "?" ver_match = re.match(r"v\d+\.\d+\.\d-\d+", ver) if ver_match: tag_version = ver_match.group() versions.append(tag_version) self.current_version, self.upstream_version = versions self.log_repo_info() except Exception: logging.exception(f"Git Repo {self.alias}: Initialization failure") raise finally: self.init_condition.notify_all() self.init_condition = None
def __init__(self): self.condition = Condition() self.counter = 0