def __init__(self, pubnub_instance): subscription_manager = self self._message_queue = Queue() self._consumer_event = Event() self._cancellation_event = Event() self._subscription_lock = Semaphore(1) # self._current_request_key_object = None self._heartbeat_periodic_callback = None self._reconnection_manager = TornadoReconnectionManager(pubnub_instance) super(TornadoSubscriptionManager, self).__init__(pubnub_instance) self._start_worker() class TornadoReconnectionCallback(ReconnectionCallback): def on_reconnect(self): subscription_manager.reconnect() pn_status = PNStatus() pn_status.category = PNStatusCategory.PNReconnectedCategory pn_status.error = False subscription_manager._subscription_status_announced = True subscription_manager._listener_manager.announce_status(pn_status) self._reconnection_listener = TornadoReconnectionCallback() self._reconnection_manager.set_reconnection_listener(self._reconnection_listener)
def __init__(self, max_concurrent_batches=10, block_on_send=False, block_on_response=False, max_batch_size=100, send_frequency=0.25, user_agent_addition=''): if not has_tornado: raise ImportError('TornadoTransmission requires tornado, but it was not found.') self.block_on_send = block_on_send self.block_on_response = block_on_response self.max_batch_size = max_batch_size self.send_frequency = send_frequency user_agent = "libhoney-py/" + VERSION if user_agent_addition: user_agent += " " + user_agent_addition self.http_client = AsyncHTTPClient( force_instance=True, defaults=dict(user_agent=user_agent)) # libhoney adds events to the pending queue for us to send self.pending = Queue(maxsize=1000) # we hand back responses from the API on the responses queue self.responses = Queue(maxsize=2000) self.batch_data = {} self.sd = statsd.StatsClient(prefix="libhoney") self.batch_sem = Semaphore(max_concurrent_batches)
def __init__(self, device_server, stream, address): self.fw_version = 0.0 self.recv_msg_cond = Condition() self.recv_msg = {} self.send_msg_sem = Semaphore(1) self.pending_request_cnt = 0 self.device_server = device_server self.stream = stream self.address = address self.stream.set_nodelay(True) self.timeout_handler_onlinecheck = None self.timeout_handler_offline = None self.killed = False self.sn = "" self.private_key = "" self.node_id = 0 self.iv = None self.cipher_down = None self.cipher_up = None #self.state_waiters = [] #self.state_happened = [] self.event_waiters = [] self.event_happened = [] self.ota_ing = False self.ota_notify_done_future = None self.post_ota = False self.online_status = True
def setup_handler( pairs_path, nonpairs_path, lang_names, missing_freqs_path, timeout, max_pipes_per_pair, min_pipes_per_pair, max_users_per_pipe, max_idle_secs, restart_pipe_after, max_doc_pipes, verbosity=0, scale_mt_logs=False, memory=1000, apy_keys=None, ): global missing_freqs_db if missing_freqs_path: missing_freqs_db = missingdb.MissingDb(missing_freqs_path, memory) handler = BaseHandler handler.lang_names = lang_names handler.timeout = timeout handler.max_pipes_per_pair = max_pipes_per_pair handler.min_pipes_per_pair = min_pipes_per_pair handler.max_users_per_pipe = max_users_per_pipe handler.max_idle_secs = max_idle_secs handler.restart_pipe_after = restart_pipe_after handler.scale_mt_logs = scale_mt_logs handler.verbosity = verbosity handler.doc_pipe_sem = Semaphore(max_doc_pipes) handler.api_keys_conf = apy_keys modes = search_path(pairs_path, verbosity=verbosity) if nonpairs_path: src_modes = search_path(nonpairs_path, include_pairs=False, verbosity=verbosity) for mtype in modes: modes[mtype] += src_modes[mtype] for mtype in modes: logging.info('%d %s modes found', len(modes[mtype]), mtype) for path, lang_src, lang_trg in modes['pair']: handler.pairs['%s-%s' % (lang_src, lang_trg)] = path for dirpath, modename, lang_pair in modes['analyzer']: handler.analyzers[lang_pair] = (dirpath, modename) for dirpath, modename, lang_pair in modes['generator']: handler.generators[lang_pair] = (dirpath, modename) for dirpath, modename, lang_pair in modes['tagger']: handler.taggers[lang_pair] = (dirpath, modename) for dirpath, modename, lang_src in modes['spell']: if (any(lang_src == elem[2] for elem in modes['tokenise'])): handler.spellers[lang_src] = (dirpath, modename) handler.init_pairs_graph() handler.init_paths()
def __singleton_init__(self, socket_path, io_loop=None): super(IPCMessageSubscriber, self).__singleton_init__( socket_path, io_loop=io_loop) self._read_sync_future = None self._read_stream_future = None self._sync_ioloop_running = False self.saved_data = [] self._sync_read_in_progress = Semaphore()
def __init__(self, db_uri, project_id, admin_uid, client_id, client_secret, api_key, api_rq_interval, domain, secure, static_uri, static_path, thread_count, crawler_config, **kwargs): self._log = extdlog.getLogger(self.__class__.__name__) # Database connection self._db = Database(db_uri, log=self._log.getChild('db')) # Session management connection self._pool = WorkerPool(thread_count) self._hasher = ImageHasher(self._log.getChild('hasher'), self._pool) AsyncHTTPClient.configure( None, defaults=dict( user_agent= "HADSH/0.0.1 (https://hackaday.io/project/29161-hackadayio-spambot-hunter-project)" )) self._api = HackadayAPI(client_id=client_id, client_secret=client_secret, api_key=api_key, rqlim_time=api_rq_interval, client=AsyncHTTPClient(), log=self._log.getChild('api')) self._crawler = Crawler(project_id, admin_uid, self._db, self._api, self._hasher, self._log.getChild('crawler'), config=crawler_config) self._resizer = ImageResizer(self._log.getChild('resizer'), self._pool) self._domain = domain self._secure = secure self._classify_sem = Semaphore(1) self._crypt_context = CryptContext(['argon2', 'scrypt', 'bcrypt']) # Initialise traits init_traits(self, self._log.getChild('trait')) super(HADSHApp, self).__init__([ (r"/", RootHandler), (r"/login", LoginHandler), (r"/avatar/([0-9]+)", AvatarHandler), (r"/avatar/(average_hash|dhash|phash|whash|sha512)/([0-9]+)", \ AvatarHashHandler), (r"/user/([0-9]+)", UserHandler), (r"/callback", CallbackHandler), (r"/classify/([0-9]+)", ClassifyHandler), (r"/data/newcomers.json", NewcomerDataHandler), (r"/data/legit.json", LegitUserDataHandler), (r"/data/suspect.json", SuspectUserDataHandler), (r"/data/admin.json", AdminUserDataHandler), (r"/authorize", RedirectHandler, { "url": self._api.auth_uri }), ], static_url_prefix=static_uri, static_path=static_path, **kwargs)
def __init__(self, trunk_id, name, max_lines, tag, direction): self.id = trunk_id self.name = name self.max_lines = trun self.tag = tag self.direction = direction self.lines = { l: {} for l in range(0, max_lines) } self.semaphore = Semaphore(1) self.callers = {} self.counters =
def __init__(self, start_link=None): self._init_defaults() # Now load the config file to override defaults self._load_config() if start_link: self.start_link = start_link if not self.start_link: raise SystemExit("No start link is provided, exiting now...") links.put(self.start_link) self.semaphore = Semaphore(self.workers_limit)
def __init__(self, prespawn_count, kernel_manager): # Make sure there's at least one kernel as a delegate if not prespawn_count: prespawn_count = 1 super(ManagedKernelPool, self).__init__(prespawn_count, kernel_manager) self.kernel_clients = {} self.on_recv_funcs = {} self.kernel_pool = [] kernel_ids = self.kernel_manager.list_kernel_ids() self.kernel_semaphore = Semaphore(len(kernel_ids)) # Create clients and iopub handlers for prespawned kernels for kernel_id in kernel_ids: self.kernel_clients[kernel_id] = kernel_manager.get_kernel( kernel_id).client() self.kernel_pool.append(kernel_id) iopub = self.kernel_manager.connect_iopub(kernel_id) iopub.on_recv(self.create_on_reply(kernel_id))
class AsyncModbusGeneratorClient(AsyncModbusSerialClient): def __init__(self, method='ascii', **kwargs): super(AsyncModbusGeneratorClient, self).__init__(method=method, **kwargs) self.sem = Semaphore(1) @gen.coroutine def read_input_registers(self, address, count=1, **kwargs): fut_result = Future() request = ReadInputRegistersRequest(address, count, **kwargs) yield self.sem.acquire() try: res = self.execute(request) res.addCallback(fut_result.set_result) yield fut_result finally: self.sem.release() raise gen.Return(fut_result.result()) @gen.coroutine def write_coil(self, address, value, **kwargs): fut_result = Future() request = WriteSingleCoilRequest(address, value, **kwargs) yield self.sem.acquire() try: res = self.execute(request) res.addCallback(fut_result.set_result) yield fut_result finally: self.sem.release() raise gen.Return(fut_result.result()) @gen.coroutine def write_register(self, address, value, **kwargs): fut_result = Future() request = WriteSingleRegisterRequest(address, value, **kwargs) yield self.sem.acquire() try: res = self.execute(request) res.addCallback(fut_result.set_result) yield fut_result finally: self.sem.release() raise gen.Return(fut_result.result())
class Worker(object): def __init__(self, max_queue=0, io_loop=None): self.io_loop = io_loop or IOLoop.current() self.active = {} self.sem = Semaphore(value=max_queue) self.log = logging.getLogger("pizzadelivery.downloader") def _add_to_active(self, task, future): self.active[self.task_to_id(task)] = future def _in_active(self, task): return self.task_to_id(task) in self.active def _remove_from_active(self, task): del self.active[self.task_to_id(task)] def _get_future_for_task(self, task): return self.active[self.task_to_id(task)] def enqueue(self, task): if self._in_active(task): future = concurrent.Future() concurrent.chain_future(self._get_future_for_task(task), future) return future future = concurrent.Future() self._add_to_active(task, future) concurrent.chain_future(self._do(task), future) return future @gen.coroutine def _do(self, task): assert self._in_active(task) try: with (yield self.sem.acquire()): res = yield gen.maybe_future(self.do(task)) raise gen.Return(res) finally: self._remove_from_active(task) @gen.coroutine def do(self, task): raise NotImplementedError # pragma: no cover def task_to_id(self, task): return task
def extract_proxies_async(requests_proxies): """ :rtype: {'http://123.169.238.33:8888', ...} """ SEMA = Semaphore(CONCURRENT_NUM) @gen.coroutine def worker(instance, idx, item, requests_proxies): with (yield SEMA.acquire()): proxies = instance.parse_proxies_async(item) yield instance.async_http(proxies, idx, requests_proxies) instance = ExtractProxies.instance() raw_proxies = instance.get_proxies() if raw_proxies is not None: yield [worker(instance, idx, item, requests_proxies) \ for idx, item in enumerate(raw_proxies)]
def __init__(self, prespawn_count, kernel_manager): # Make sure there's at least one kernel as a delegate if not prespawn_count: prespawn_count = 1 super(ManagedKernelPool, self).__init__(prespawn_count, kernel_manager) self.kernel_clients = {} self.on_recv_funcs = {} self.kernel_pool = [] kernel_ids = self.kernel_manager.list_kernel_ids() self.kernel_semaphore = Semaphore(len(kernel_ids)) # Create clients and iopub handlers for prespawned kernels for kernel_id in kernel_ids: self.kernel_clients[kernel_id] = kernel_manager.get_kernel(kernel_id).client() self.kernel_pool.append(kernel_id) iopub = self.kernel_manager.connect_iopub(kernel_id) iopub.on_recv(self.create_on_reply(kernel_id))
def __init__(self, prespawn_count, kernel_manager): if prespawn_count is None: prespawn_count = 0 self.kernel_clients = {} self.on_recv_funcs = {} self.kernel_manager = kernel_manager self.pool_index = 0 self.kernel_pool = [] self.kernel_semaphore = Semaphore(prespawn_count) for _ in range(prespawn_count): if self.kernel_manager.parent.seed_notebook: kernel_id = kernel_manager.start_kernel(kernel_name=self.kernel_manager.parent.seed_notebook['metadata']['kernelspec']['name']) else: kernel_id = kernel_manager.start_kernel() self.kernel_clients[kernel_id] = kernel_manager.get_kernel(kernel_id).client() self.kernel_pool.append(kernel_id) iopub = self.kernel_manager.connect_iopub(kernel_id) iopub.on_recv(self.create_on_reply(kernel_id))
def get_resized(self, gallery, photo, width=None, height=None, quality=60, rotation=0.0, img_format=None, orientation=0): """ Retrieve the given photo in a resized format. """ # Determine the path to the original file. orig_node = self._fs_node.join_node(gallery, photo) if img_format is None: # Detect from original file and quality setting. with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m: mime_type = m.id_filename(orig_node.abs_path) self._log.debug('%s/%s detected format %s', gallery, photo, mime_type) if mime_type == 'image/gif': img_format = ImageFormat.GIF else: if quality == 100: # Assume PNG img_format = ImageFormat.PNG else: # Assume JPEG img_format = ImageFormat.JPEG else: # Use the format given by the user img_format = ImageFormat(img_format) self._log.debug('%s/%s using %s format', gallery, photo, img_format.name) # Sanitise dimensions given by user. width, height = self.get_dimensions(gallery, photo, width, height) self._log.debug('%s/%s target dimensions %d by %d', gallery, photo, width, height) # Determine where the file would be cached (cache_dir, cache_name) = self._get_cache_name(gallery, photo, width,height, quality, rotation, img_format) # Do we have this file? data = self._read_cache(orig_node, cache_dir, cache_name) if data is not None: raise Return((img_format, cache_name, data)) # Locate the lock for this photo. mutex_key = (gallery, photo, width, height, quality, rotation, img_format) try: mutex = self._mutexes[mutex_key] except KeyError: mutex = Semaphore(1) self._mutexes[mutex_key] = mutex resize_args = (gallery, photo, width, height, quality, rotation, img_format.value, orientation) try: self._log.debug('%s/%s waiting for mutex', gallery, photo) yield mutex.acquire() # We have the semaphore, call our resize routine. self._log.debug('%s/%s retrieving resized image (args=%s)', gallery, photo, resize_args) (img_format, file_name, file_data) = yield self._pool.apply( func=self._do_resize, args=resize_args) raise Return((img_format, file_name, file_data)) except Return: raise except: self._log.exception('Error resizing photo; gallery: %s, photo: %s, '\ 'width: %d, height: %d, quality: %f, rotation: %f, format: %s', gallery, photo, width, height, quality, rotation, img_format) raise finally: mutex.release()
class ManagedKernelPool(KernelPool): """Spawns a pool of kernels that are treated as identical delegates for future requests. Manages access to individual kernels using a borrower/lender pattern. Cleans them all up when shut down. Parameters ---------- prespawn_count Number of kernels to spawn immediately kernel_manager Kernel manager instance Attributes ---------- kernel_clients : dict Map of kernel IDs to client instances for communicating with them on_recv_funcs : dict Map of kernel IDs to iopub callback functions kernel_pool : list List of available delegate kernel IDs kernel_semaphore : tornado.locks.Semaphore Semaphore that controls access to the kernel pool """ def __init__(self, prespawn_count, kernel_manager): # Make sure there's at least one kernel as a delegate if not prespawn_count: prespawn_count = 1 super(ManagedKernelPool, self).__init__(prespawn_count, kernel_manager) self.kernel_clients = {} self.on_recv_funcs = {} self.kernel_pool = [] kernel_ids = self.kernel_manager.list_kernel_ids() self.kernel_semaphore = Semaphore(len(kernel_ids)) # Create clients and iopub handlers for prespawned kernels for kernel_id in kernel_ids: self.kernel_clients[kernel_id] = kernel_manager.get_kernel( kernel_id).client() self.kernel_pool.append(kernel_id) iopub = self.kernel_manager.connect_iopub(kernel_id) iopub.on_recv(self.create_on_reply(kernel_id)) @gen.coroutine def acquire(self): """Gets a kernel client and removes it from the available pool of clients. Returns ------- tuple Kernel client instance, kernel ID """ yield self.kernel_semaphore.acquire() kernel_id = self.kernel_pool[0] del self.kernel_pool[0] raise gen.Return((self.kernel_clients[kernel_id], kernel_id)) def release(self, kernel_id): """Puts a kernel back into the pool of kernels available to handle requests. Parameters ---------- kernel_id : str Kernel to return to the pool """ self.kernel_pool.append(kernel_id) self.kernel_semaphore.release() def _on_reply(self, kernel_id, msg_list): """Invokes the iopub callback registered for the `kernel_id` and passes it a deserialized list of kernel messsages. Parameters ---------- kernel_id : str Kernel that sent the reply msg_list : list List of 0mq messages """ idents, msg_list = self.kernel_clients[ kernel_id].session.feed_identities(msg_list) msg = self.kernel_clients[kernel_id].session.deserialize(msg_list) self.on_recv_funcs[kernel_id](msg) def create_on_reply(self, kernel_id): """Creates an anonymous function to handle reply messages from the kernel. Parameters ---------- kernel_id Kernel to listen to Returns ------- function Callback function taking a kernel ID and 0mq message list """ return lambda msg_list: self._on_reply(kernel_id, msg_list) def on_recv(self, kernel_id, func): """Registers a callback function for iopub messages from a particular kernel. This is needed to avoid having multiple callbacks per kernel client. Parameters ---------- kernel_id Kernel from which to receive iopub messages func Callback function to use for kernel iopub messages """ self.on_recv_funcs[kernel_id] = func def shutdown(self): """Shuts down all kernels and their clients. """ for kid in self.kernel_clients: self.kernel_clients[kid].stop_channels() self.kernel_manager.shutdown_kernel(kid, now=True) # Any remaining kernels that were not created for our pool should be shutdown super(ManagedKernelPool, self).shutdown()
#!urs/bin/env python #coding:utf-8 # Semaphore:一个信号量管理着代表release调用次数减去acquire的调用次数的计数器加一个 # 初始值。如果必要的话,acquire方法将会阻塞,直到它可以返回,而不使该计数器成为负值。 # 信号量限制访问共享资源,为了允许两个worker同时获得权限,代码如下。 from tornado import gen from tornado.ioloop import IOLoop from tornado.locks import Semaphore sem = Semaphore(2) @gen.coroutine def worker(worker_id): with (yield sem.acquire()): # acquire()是一个上下文管理器 print("Worker %d is working" % worker_id) print("Worker %d is done" % worker_id) sem.release() # yield sem.acquire() # try: # print('Worker %d is working' % worker_id) # # yield use_some_resource() # finally: # print('Worker %d is done' % worker_id) # sem.release() @gen.coroutine
class TornadoSubscriptionManager(SubscriptionManager): def __init__(self, pubnub_instance): self._message_queue = Queue() self._consumer_event = Event() self._subscription_lock = Semaphore(1) # self._current_request_key_object = None self._heartbeat_periodic_callback = None self._cancellation_event = None super(TornadoSubscriptionManager, self).__init__(pubnub_instance) self._start_worker() def _set_consumer_event(self): self._consumer_event.set() def _message_queue_put(self, message): self._message_queue.put(message) def _start_worker(self): self._consumer = TornadoSubscribeMessageWorker(self._pubnub, self._listener_manager, self._message_queue, self._consumer_event) run = stack_context.wrap(self._consumer.run) self._pubnub.ioloop.spawn_callback(run) def reconnect(self): self._should_stop = False self._pubnub.ioloop.add_callback(self._start_subscribe_loop) self._register_heartbeat_timer() @tornado.gen.coroutine def _start_subscribe_loop(self): try: self._stop_subscribe_loop() yield self._subscription_lock.acquire() self._cancellation_event = Event() combined_channels = self._subscription_state.prepare_channel_list(True) combined_groups = self._subscription_state.prepare_channel_group_list(True) if len(combined_channels) == 0 and len(combined_groups) == 0: return envelope_future = Subscribe(self._pubnub) \ .channels(combined_channels).channel_groups(combined_groups) \ .timetoken(self._timetoken).region(self._region) \ .filter_expression(self._pubnub.config.filter_expression) \ .cancellation_event(self._cancellation_event) \ .future() wi = tornado.gen.WaitIterator( envelope_future, self._cancellation_event.wait()) while not wi.done(): try: result = yield wi.next() except Exception as e: logger.error(e) raise else: if wi.current_future == envelope_future: envelope = result elif wi.current_future == self._cancellation_event.wait(): break self._handle_endpoint_call(envelope.result, envelope.status) self._start_subscribe_loop() except PubNubTornadoException as e: if e.status is not None and e.status.category == PNStatusCategory.PNTimeoutCategory: self._pubnub.ioloop.add_callback(self._start_subscribe_loop) else: self._listener_manager.announce_status(e.status) except Exception as e: logger.error(e) raise finally: self._cancellation_event.set() yield tornado.gen.moment self._cancellation_event = None self._subscription_lock.release() def _stop_subscribe_loop(self): if self._cancellation_event is not None: self._cancellation_event.set() def _stop_heartbeat_timer(self): if self._heartbeat_periodic_callback is not None: self._heartbeat_periodic_callback.stop() def _register_heartbeat_timer(self): super(TornadoSubscriptionManager, self)._register_heartbeat_timer() self._heartbeat_periodic_callback = PeriodicCallback( stack_context.wrap(self._perform_heartbeat_loop), self._pubnub.config.heartbeat_interval * TornadoSubscriptionManager.HEARTBEAT_INTERVAL_MULTIPLIER, self._pubnub.ioloop) self._heartbeat_periodic_callback.start() @tornado.gen.coroutine def _perform_heartbeat_loop(self): if self._heartbeat_call is not None: # TODO: cancel call pass cancellation_event = Event() state_payload = self._subscription_state.state_payload() presence_channels = self._subscription_state.prepare_channel_list(False) presence_groups = self._subscription_state.prepare_channel_group_list(False) if len(presence_channels) == 0 and len(presence_groups) == 0: return try: envelope = yield self._pubnub.heartbeat() \ .channels(presence_channels) \ .channel_groups(presence_groups) \ .state(state_payload) \ .cancellation_event(cancellation_event) \ .future() heartbeat_verbosity = self._pubnub.config.heartbeat_notification_options if envelope.status.is_error: if heartbeat_verbosity == PNHeartbeatNotificationOptions.ALL or \ heartbeat_verbosity == PNHeartbeatNotificationOptions.ALL: self._listener_manager.announce_stateus(envelope.status) else: if heartbeat_verbosity == PNHeartbeatNotificationOptions.ALL: self._listener_manager.announce_stateus(envelope.status) except PubNubTornadoException: pass # TODO: check correctness # if e.status is not None and e.status.category == PNStatusCategory.PNTimeoutCategory: # self._start_subscribe_loop() # else: # self._listener_manager.announce_status(e.status) finally: cancellation_event.set() @tornado.gen.coroutine def _send_leave(self, unsubscribe_operation): envelope = yield Leave(self._pubnub) \ .channels(unsubscribe_operation.channels) \ .channel_groups(unsubscribe_operation.channel_groups).future() self._listener_manager.announce_status(envelope.status)
def cull_idle( url, api_token, inactive_limit, protected_users, cull_users=False, max_age=0, concurrency=10, ): """Shutdown idle single-user servers If cull_users, inactive *users* will be deleted as well. """ auth_header = { "Authorization": "token %s" % api_token, } req = HTTPRequest( url=url + "/users", headers=auth_header, ) now = datetime.now(timezone.utc) client = AsyncHTTPClient() if concurrency: semaphore = Semaphore(concurrency) @coroutine def fetch(req): """client.fetch wrapped in a semaphore to limit concurrency""" yield semaphore.acquire() try: return (yield client.fetch(req)) finally: yield semaphore.release() else: fetch = client.fetch resp = yield fetch(req) users = json.loads(resp.body.decode("utf8", "replace")) futures = [] @coroutine def handle_server(user, server_name, server): """Handle (maybe) culling a single server Returns True if server is now stopped (user removable), False otherwise. """ log_name = user["name"] if server_name: log_name = "%s/%s" % (user["name"], server_name) if server.get("pending"): app_log.warning("Not culling server %s with pending %s", log_name, server["pending"]) return False # jupyterhub < 0.9 defined 'server.url' once the server was ready # as an *implicit* signal that the server was ready. # 0.9 adds a dedicated, explicit 'ready' field. # By current (0.9) definitions, servers that have no pending # events and are not ready shouldn't be in the model, # but let's check just to be safe. if not server.get("ready", bool(server["url"])): app_log.warning("Not culling not-ready not-pending server %s: %s", log_name, server) return False if server.get("started"): age = now - parse_date(server["started"]) else: # started may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if server["last_activity"]: inactive = now - parse_date(server["last_activity"]) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'started' field which is never None # for running servers inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling server %s (inactive for %s)", log_name, format_td(inactive)) if max_age and not should_cull: # only check started if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive), ) should_cull = True if not should_cull: app_log.debug( "Not culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive), ) return False if server_name: # culling a named server delete_url = url + "/users/%s/servers/%s" % ( quote(user["name"]), quote(server["name"]), ) else: delete_url = url + "/users/%s/server" % quote(user["name"]) req = HTTPRequest( url=delete_url, method="DELETE", headers=auth_header, ) resp = yield fetch(req) if resp.code == 202: app_log.warning( "Server %s is slow to stop", log_name, ) # return False to prevent culling user with pending shutdowns return False return True @coroutine def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. # Hub doesn't allow deleting users with running servers. # jupyterhub 0.9 always provides a 'servers' model. # 0.8 only does this when named servers are enabled. if "servers" in user: servers = user["servers"] else: # jupyterhub < 0.9 without named servers enabled. # create servers dict with one entry for the default server # from the user model. # only if the server is running. servers = {} if user["server"]: servers[""] = { "last_activity": user["last_activity"], "pending": user["pending"], "url": user["server"], } server_futures = [ handle_server(user, server_name, server) for server_name, server in servers.items() ] results = yield multi(server_futures) if not cull_users: return # some servers are still running, cannot cull users still_alive = len(results) - sum(results) if still_alive: app_log.debug( "Not culling user %s with %i servers still alive", user["name"], still_alive, ) return False should_cull = False if user.get("created"): age = now - parse_date(user["created"]) else: # created may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if user["last_activity"]: inactive = now - parse_date(user["last_activity"]) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'created' field which is never None inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling user %s (inactive for %s)", user["name"], inactive) if max_age and not should_cull: # only check created if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling user %s (age: %s, inactive for %s)", user["name"], format_td(age), format_td(inactive), ) should_cull = True if not should_cull: app_log.debug( "Not culling user %s (created: %s, last active: %s)", user["name"], format_td(age), format_td(inactive), ) return False req = HTTPRequest( url=url + "/users/%s" % user["name"], method="DELETE", headers=auth_header, ) yield fetch(req) return True p_users = [] # Don't kill protected servers if protected_users is not None: # Expects a list of eithers users or files that contains users for protected in protected_users.split(","): if os.path.exists(protected): db_path = protected try: app_log.info("Cull, checking db {} " "for users".format(db_path)) file_users = [] with open(db_path, "r") as db: file_users = [ user.rstrip("\n").rstrip("\r\n") for user in db ] p_users.extend(file_users) except IOError as err: app_log.error("Cull, tried to open db file {}," "Failed {}".format(db_path, err)) else: p_users.append(protected) users = [user for user in users if user["name"] not in p_users] for user in users: futures.append((user["name"], handle_user(user))) for (name, f) in futures: try: result = yield f except Exception: app_log.exception("Error processing %s", name) else: if result: app_log.debug("Finished culling %s", name)
def __init__(self, max_queue=0, io_loop=None): self.io_loop = io_loop or IOLoop.current() self.active = {} self.sem = Semaphore(value=max_queue) self.log = logging.getLogger("pizzadelivery.downloader")
def cull_idle(url, api_token, inactive_limit, cull_users=False, max_age=0, concurrency=10): """Shutdown idle single-user servers If cull_users, inactive *users* will be deleted as well. """ auth_header = {'Authorization': 'token %s' % api_token} req = HTTPRequest(url=url + '/users', headers=auth_header) now = datetime.now(timezone.utc) client = AsyncHTTPClient() if concurrency: semaphore = Semaphore(concurrency) @coroutine def fetch(req): """client.fetch wrapped in a semaphore to limit concurrency""" yield semaphore.acquire() try: return (yield client.fetch(req)) finally: yield semaphore.release() else: fetch = client.fetch resp = yield fetch(req) users = json.loads(resp.body.decode('utf8', 'replace')) futures = [] @coroutine def handle_server(user, server_name, server): """Handle (maybe) culling a single server Returns True if server is now stopped (user removable), False otherwise. """ log_name = user['name'] if server_name: log_name = '%s/%s' % (user['name'], server_name) if server.get('pending'): app_log.warning("Not culling server %s with pending %s", log_name, server['pending']) return False # jupyterhub < 0.9 defined 'server.url' once the server was ready # as an *implicit* signal that the server was ready. # 0.9 adds a dedicated, explicit 'ready' field. # By current (0.9) definitions, servers that have no pending # events and are not ready shouldn't be in the model, # but let's check just to be safe. if not server.get('ready', bool(server['url'])): app_log.warning("Not culling not-ready not-pending server %s: %s", log_name, server) return False if server.get('started'): age = now - parse_date(server['started']) else: # started may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if server['last_activity']: inactive = now - parse_date(server['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'started' field which is never None # for running servers inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling server %s (inactive for %s)", log_name, format_td(inactive)) if max_age and not should_cull: # only check started if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive), ) should_cull = True if not should_cull: app_log.debug( "Not culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive), ) return False if server_name: # culling a named server delete_url = url + "/users/%s/servers/%s" % ( quote(user['name']), quote(server['name']), ) else: delete_url = url + '/users/%s/server' % quote(user['name']) req = HTTPRequest(url=delete_url, method='DELETE', headers=auth_header) resp = yield fetch(req) if resp.code == 202: app_log.warning("Server %s is slow to stop", log_name) # return False to prevent culling user with pending shutdowns return False return True @coroutine def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. # Hub doesn't allow deleting users with running servers. # jupyterhub 0.9 always provides a 'servers' model. # 0.8 only does this when named servers are enabled. if 'servers' in user: servers = user['servers'] else: # jupyterhub < 0.9 without named servers enabled. # create servers dict with one entry for the default server # from the user model. # only if the server is running. servers = {} if user['server']: servers[''] = { 'last_activity': user['last_activity'], 'pending': user['pending'], 'url': user['server'], } server_futures = [ handle_server(user, server_name, server) for server_name, server in servers.items() ] results = yield multi(server_futures) if not cull_users: return # some servers are still running, cannot cull users still_alive = len(results) - sum(results) if still_alive: app_log.debug( "Not culling user %s with %i servers still alive", user['name'], still_alive, ) return False should_cull = False if user.get('created'): age = now - parse_date(user['created']) else: # created may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if user['last_activity']: inactive = now - parse_date(user['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'created' field which is never None inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling user %s (inactive for %s)", user['name'], inactive) if max_age and not should_cull: # only check created if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling user %s (age: %s, inactive for %s)", user['name'], format_td(age), format_td(inactive), ) should_cull = True if not should_cull: app_log.debug( "Not culling user %s (created: %s, last active: %s)", user['name'], format_td(age), format_td(inactive), ) return False req = HTTPRequest(url=url + '/users/%s' % user['name'], method='DELETE', headers=auth_header) yield fetch(req) return True for user in users: futures.append((user['name'], handle_user(user))) for (name, f) in futures: try: result = yield f except Exception: app_log.exception("Error processing %s", name) else: if result: app_log.debug("Finished culling %s", name)
class ManagedKernelPool(KernelPool): ''' Spawns a pool of kernels. Manages access to individual kernels using a borrower/lender pattern. Cleans them all up when shut down. ''' def __init__(self, prespawn_count, kernel_manager): # Make sure there's at least one kernel as a delegate if not prespawn_count: prespawn_count = 1 super(ManagedKernelPool, self).__init__(prespawn_count, kernel_manager) self.kernel_clients = {} self.on_recv_funcs = {} self.pool_index = 0 self.kernel_pool = [] kernel_ids = self.kernel_manager.list_kernel_ids() self.kernel_semaphore = Semaphore(len(kernel_ids)) # Connect to any prespawned kernels for kernel_id in kernel_ids: self.kernel_clients[kernel_id] = kernel_manager.get_kernel(kernel_id).client() self.kernel_pool.append(kernel_id) iopub = self.kernel_manager.connect_iopub(kernel_id) iopub.on_recv(self.create_on_reply(kernel_id)) @gen.coroutine def acquire(self): ''' Returns a kernel client and id for use and removes the kernel the resource pool. Kernels must be returned via the release method. :return: Returns a kernel client and a kernel id ''' yield self.kernel_semaphore.acquire() kernel_id = self.kernel_pool[0] del self.kernel_pool[0] raise gen.Return((self.kernel_clients[kernel_id], kernel_id)) def release(self, kernel_id): ''' Returns a kernel back to the resource pool. :param kernel_id: Id of the kernel to return to the pool ''' self.kernel_pool.append(kernel_id) self.kernel_semaphore.release() def _on_reply(self, kernel_id, msg_list): idents, msg_list = self.kernel_clients[kernel_id].session.feed_identities(msg_list) msg = self.kernel_clients[kernel_id].session.deserialize(msg_list) self.on_recv_funcs[kernel_id](msg) def create_on_reply(self, kernel_id): ''' The lambda is used to handle a specific reply per kernel and provide a unique stack scope per invocation. ''' return lambda msg_list: self._on_reply(kernel_id, msg_list) def on_recv(self, kernel_id, func): ''' Registers a callback for io_pub messages for a particular kernel. This is needed to avoid having multiple callbacks per kernel client. :param kernel_id: Id of the kernel :param func: Callback function to handle the message ''' self.on_recv_funcs[kernel_id] = func def shutdown(self): ''' Shuts down all kernels in the pool and in the kernel manager. ''' for kid in self.kernel_clients: self.kernel_clients[kid].stop_channels() self.kernel_manager.shutdown_kernel(kid, now=True) # Any remaining kernels that were not created for our pool should be shutdown super(ManagedKernelPool, self).shutdown()
class TornadoTransmission(): def __init__(self, max_concurrent_batches=10, block_on_send=False, block_on_response=False, max_batch_size=100, send_frequency=0.25, user_agent_addition=''): if not has_tornado: raise ImportError('TornadoTransmission requires tornado, but it was not found.') self.block_on_send = block_on_send self.block_on_response = block_on_response self.max_batch_size = max_batch_size self.send_frequency = send_frequency user_agent = "libhoney-py/" + VERSION if user_agent_addition: user_agent += " " + user_agent_addition self.http_client = AsyncHTTPClient( force_instance=True, defaults=dict(user_agent=user_agent)) # libhoney adds events to the pending queue for us to send self.pending = Queue(maxsize=1000) # we hand back responses from the API on the responses queue self.responses = Queue(maxsize=2000) self.batch_data = {} self.sd = statsd.StatsClient(prefix="libhoney") self.batch_sem = Semaphore(max_concurrent_batches) def start(self): ioloop.IOLoop.current().spawn_callback(self._sender) def send(self, ev): '''send accepts an event and queues it to be sent''' self.sd.gauge("queue_length", self.pending.qsize()) try: if self.block_on_send: self.pending.put(ev) else: self.pending.put_nowait(ev) self.sd.incr("messages_queued") except QueueFull: response = { "status_code": 0, "duration": 0, "metadata": ev.metadata, "body": "", "error": "event dropped; queue overflow", } if self.block_on_response: self.responses.put(response) else: try: self.responses.put_nowait(response) except QueueFull: # if the response queue is full when trying to add an event # queue is full response, just skip it. pass self.sd.incr("queue_overflow") # We're using the older decorator/yield model for compatibility with # Python versions before 3.5. # See: http://www.tornadoweb.org/en/stable/guide/coroutines.html#python-3-5-async-and-await @gen.coroutine def _sender(self): '''_sender is the control loop that pulls events off the `self.pending` queue and submits batches for actual sending. ''' events = [] last_flush = time.time() while True: try: ev = yield self.pending.get(timeout=self.send_frequency) if ev is None: # signals shutdown yield self._flush(events) return events.append(ev) if (len(events) > self.max_batch_size or time.time() - last_flush > self.send_frequency): yield self._flush(events) events = [] except TimeoutError: yield self._flush(events) events = [] last_flush = time.time() @gen.coroutine def _flush(self, events): if not events: return for dest, group in group_events_by_destination(events).items(): yield self._send_batch(dest, group) @gen.coroutine def _send_batch(self, destination, events): ''' Makes a single batch API request with the given list of events. The `destination` argument contains the write key, API host and dataset name used to build the request.''' start = time.time() status_code = 0 try: # enforce max_concurrent_batches yield self.batch_sem.acquire() url = urljoin(urljoin(destination.api_host, "/1/batch/"), destination.dataset) payload = [] for ev in events: event_time = ev.created_at.isoformat() if ev.created_at.tzinfo is None: event_time += "Z" payload.append({ "time": event_time, "samplerate": ev.sample_rate, "data": ev.fields()}) req = HTTPRequest( url, method='POST', headers={ "X-Honeycomb-Team": destination.writekey, "Content-Type": "application/json", }, body=json.dumps(payload, default=json_default_handler), ) self.http_client.fetch(req, self._response_callback) # store the events that were sent so we can process responses later # it is important that we delete these eventually, or we'll run into memory issues self.batch_data[req] = {"start": start, "events": events} except Exception as e: # Catch all exceptions and hand them to the responses queue. self._enqueue_errors(status_code, e, start, events) finally: self.batch_sem.release() def _enqueue_errors(self, status_code, error, start, events): for ev in events: self.sd.incr("send_errors") self._enqueue_response(status_code, "", error, start, ev.metadata) def _response_callback(self, resp): # resp.request should be the same HTTPRequest object built by _send_batch # and mapped to values in batch_data events = self.batch_data[resp.request]["events"] start = self.batch_data[resp.request]["start"] try: status_code = resp.code resp.rethrow() statuses = [d["status"] for d in json.loads(resp.body)] for ev, status in zip(events, statuses): self._enqueue_response(status, "", None, start, ev.metadata) self.sd.incr("messages_sent") except Exception as e: self._enqueue_errors(status_code, e, start, events) self.sd.incr("send_errors") finally: # clean up the data for this batch del self.batch_data[resp.request] def _enqueue_response(self, status_code, body, error, start, metadata): resp = { "status_code": status_code, "body": body, "error": error, "duration": (time.time() - start) * 1000, "metadata": metadata } if self.block_on_response: self.responses.put(resp) else: try: self.responses.put_nowait(resp) except QueueFull: pass def close(self): '''call close to send all in-flight requests and shut down the senders nicely. Times out after max 20 seconds per sending thread plus 10 seconds for the response queue''' try: self.pending.put(None, 10) except QueueFull: pass # signal to the responses queue that nothing more is coming. try: self.responses.put(None, 10) except QueueFull: pass def get_response_queue(self): ''' return the responses queue on to which will be sent the response objects from each event send''' return self.responses
class KernelPool(object): ''' A class to maintain a pool of kernel and control access to the individual kernels. Kernels are protected by a borrower/lender pattern. ''' def __init__(self, prespawn_count, kernel_manager): if prespawn_count is None: prespawn_count = 0 self.kernel_clients = {} self.on_recv_funcs = {} self.kernel_manager = kernel_manager self.pool_index = 0 self.kernel_pool = [] self.kernel_semaphore = Semaphore(prespawn_count) for _ in range(prespawn_count): if self.kernel_manager.parent.seed_notebook: kernel_id = kernel_manager.start_kernel(kernel_name=self.kernel_manager.parent.seed_notebook['metadata']['kernelspec']['name']) else: kernel_id = kernel_manager.start_kernel() self.kernel_clients[kernel_id] = kernel_manager.get_kernel(kernel_id).client() self.kernel_pool.append(kernel_id) iopub = self.kernel_manager.connect_iopub(kernel_id) iopub.on_recv(self.create_on_reply(kernel_id)) @gen.coroutine def acquire(self): ''' Returns a kernel client and id for use and removes the kernel the resource pool. Kernels must be returned via the release method. :return:Returns a kernel client and a kernel id ''' yield self.kernel_semaphore.acquire() kernel_id = self.kernel_pool[0] del self.kernel_pool[0] raise gen.Return((self.kernel_clients[kernel_id], kernel_id)) def release(self, kernel_id): ''' Returns a kernel back to the resource pool. :param kernel_id: Id of the kernel to return to the pool ''' self.kernel_pool.append(kernel_id) self.kernel_semaphore.release() def _on_reply(self, kernel_id, msg_list): idents, msg_list = self.kernel_clients[kernel_id].session.feed_identities(msg_list) msg = self.kernel_clients[kernel_id].session.deserialize(msg_list) self.on_recv_funcs[kernel_id](msg) def create_on_reply(self, kernel_id): ''' The lambda is used to handle a specific reply per kernel and provide a unique stack scope per invocation. ''' return lambda msg_list: self._on_reply(kernel_id, msg_list) def on_recv(self, kernel_id, func): ''' Registers a callback for io_pub messages for a particular kernel. This is needed to avoid having multiple callbacks per kernel client. :param kernel_id: Id of the kernel :param func: Callback function to handle the message ''' self.on_recv_funcs[kernel_id] = func def shutdown(self): ''' Shuts down all kernels in the pool and in the kernel manager. ''' for kid in self.kernel_clients: self.kernel_clients[kid].stop_channels() self.kernel_manager.shutdown_kernel(kid, now=True) # Any remaining kernels that were not created for our pool should be shutdown kids = self.kernel_manager.list_kernel_ids() for kid in kids: self.kernel_manager.shutdown_kernel(kid, now=True)
try: yield func(self, *args, **kwargs) except Exception as e: self.logger.error(traceback.format_exc()) if self.request.headers.get("Accept", "").startswith("application/json"): self.send_json_error() else: self.write_error(500) return return wrapper base_cache = BaseRedis() sem = Semaphore(1) def cache(prefix=None, key=None, ttl=60, hash=True, lock=True, separator=":"): """ cache装饰器 :param prefix: 指定prefix :param key: 指定key :param ttl: ttl (s) :param hash: 是否需要hash :param lock: - :param separator: key 分隔符 :return: """ key_ = key
import nfs import os import logging import time from tornado import web, gen from tornado.locks import Semaphore from tornado.httpclient import AsyncHTTPClient from framework import settings from framework.config import config MAX_BODY_SIZE = 4 * 1024.0 * 1024.0 * 1024.0 # 4GB GMT_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' AsyncHTTPClient.configure(None, max_body_size=MAX_BODY_SIZE) logger = logging.getLogger('default') semaphore = Semaphore(config.get('file_service_semaphore', 5)) class FileHandler(web.RequestHandler): @gen.coroutine def get(self): self.file_name = self.get_argument('filename') # type: str self.space_dir = nfs.join(settings.REPO_DIR, settings.REPO_ANT_SPACENAME) if not nfs.exists(self.space_dir): nfs.makedirs(self.space_dir) self.file_path = nfs.join(self.space_dir, self.file_name) lock_file_name = nfs.extsep + self.file_name + nfs.extsep + 'lock' self.lock_file = nfs.join(self.space_dir, lock_file_name) logger.info('#%d Request file: %s', id(self.request), self.file_name)
class TornadoSubscriptionManager(SubscriptionManager): def __init__(self, pubnub_instance): subscription_manager = self self._message_queue = Queue() self._consumer_event = Event() self._cancellation_event = Event() self._subscription_lock = Semaphore(1) # self._current_request_key_object = None self._heartbeat_periodic_callback = None self._reconnection_manager = TornadoReconnectionManager(pubnub_instance) super(TornadoSubscriptionManager, self).__init__(pubnub_instance) self._start_worker() class TornadoReconnectionCallback(ReconnectionCallback): def on_reconnect(self): subscription_manager.reconnect() pn_status = PNStatus() pn_status.category = PNStatusCategory.PNReconnectedCategory pn_status.error = False subscription_manager._subscription_status_announced = True subscription_manager._listener_manager.announce_status(pn_status) self._reconnection_listener = TornadoReconnectionCallback() self._reconnection_manager.set_reconnection_listener(self._reconnection_listener) def _set_consumer_event(self): self._consumer_event.set() def _message_queue_put(self, message): self._message_queue.put(message) def _start_worker(self): self._consumer = TornadoSubscribeMessageWorker(self._pubnub, self._listener_manager, self._message_queue, self._consumer_event) run = stack_context.wrap(self._consumer.run) self._pubnub.ioloop.spawn_callback(run) def reconnect(self): self._should_stop = False self._pubnub.ioloop.spawn_callback(self._start_subscribe_loop) # self._register_heartbeat_timer() def disconnect(self): self._should_stop = True self._stop_heartbeat_timer() self._stop_subscribe_loop() @tornado.gen.coroutine def _start_subscribe_loop(self): self._stop_subscribe_loop() yield self._subscription_lock.acquire() self._cancellation_event.clear() combined_channels = self._subscription_state.prepare_channel_list(True) combined_groups = self._subscription_state.prepare_channel_group_list(True) if len(combined_channels) == 0 and len(combined_groups) == 0: return envelope_future = Subscribe(self._pubnub) \ .channels(combined_channels).channel_groups(combined_groups) \ .timetoken(self._timetoken).region(self._region) \ .filter_expression(self._pubnub.config.filter_expression) \ .cancellation_event(self._cancellation_event) \ .future() canceller_future = self._cancellation_event.wait() wi = tornado.gen.WaitIterator(envelope_future, canceller_future) # iterates 2 times: one for result one for cancelled while not wi.done(): try: result = yield wi.next() except Exception as e: # TODO: verify the error will not be eaten logger.error(e) raise else: if wi.current_future == envelope_future: e = result elif wi.current_future == canceller_future: return else: raise Exception("Unexpected future resolved: %s" % str(wi.current_future)) if e.is_error(): # 599 error doesn't works - tornado use this status code # for a wide range of errors, for ex: # HTTP Server Error (599): [Errno -2] Name or service not known if e.status is not None and e.status.category == PNStatusCategory.PNTimeoutCategory: self._pubnub.ioloop.spawn_callback(self._start_subscribe_loop) return logger.error("Exception in subscribe loop: %s" % str(e)) if e.status is not None and e.status.category == PNStatusCategory.PNAccessDeniedCategory: e.status.operation = PNOperationType.PNUnsubscribeOperation self._listener_manager.announce_status(e.status) self._reconnection_manager.start_polling() self.disconnect() return else: self._handle_endpoint_call(e.result, e.status) self._pubnub.ioloop.spawn_callback(self._start_subscribe_loop) finally: self._cancellation_event.set() yield tornado.gen.moment self._subscription_lock.release() self._cancellation_event.clear() break def _stop_subscribe_loop(self): if self._cancellation_event is not None and not self._cancellation_event.is_set(): self._cancellation_event.set() def _stop_heartbeat_timer(self): if self._heartbeat_periodic_callback is not None: self._heartbeat_periodic_callback.stop() def _register_heartbeat_timer(self): super(TornadoSubscriptionManager, self)._register_heartbeat_timer() self._heartbeat_periodic_callback = PeriodicCallback( stack_context.wrap(self._perform_heartbeat_loop), self._pubnub.config.heartbeat_interval * TornadoSubscriptionManager.HEARTBEAT_INTERVAL_MULTIPLIER, self._pubnub.ioloop) self._heartbeat_periodic_callback.start() @tornado.gen.coroutine def _perform_heartbeat_loop(self): if self._heartbeat_call is not None: # TODO: cancel call pass cancellation_event = Event() state_payload = self._subscription_state.state_payload() presence_channels = self._subscription_state.prepare_channel_list(False) presence_groups = self._subscription_state.prepare_channel_group_list(False) if len(presence_channels) == 0 and len(presence_groups) == 0: return try: envelope = yield self._pubnub.heartbeat() \ .channels(presence_channels) \ .channel_groups(presence_groups) \ .state(state_payload) \ .cancellation_event(cancellation_event) \ .future() heartbeat_verbosity = self._pubnub.config.heartbeat_notification_options if envelope.status.is_error: if heartbeat_verbosity == PNHeartbeatNotificationOptions.ALL or \ heartbeat_verbosity == PNHeartbeatNotificationOptions.ALL: self._listener_manager.announce_status(envelope.status) else: if heartbeat_verbosity == PNHeartbeatNotificationOptions.ALL: self._listener_manager.announce_status(envelope.status) except PubNubTornadoException: pass # TODO: check correctness # if e.status is not None and e.status.category == PNStatusCategory.PNTimeoutCategory: # self._start_subscribe_loop() # else: # self._listener_manager.announce_status(e.status) except Exception as e: print(e) finally: cancellation_event.set() @tornado.gen.coroutine def _send_leave(self, unsubscribe_operation): envelope = yield Leave(self._pubnub) \ .channels(unsubscribe_operation.channels) \ .channel_groups(unsubscribe_operation.channel_groups).future() self._listener_manager.announce_status(envelope.status)
class HackadayAPI(object): """ Core Hackaday.io API handler. """ HAD_API_URI='https://api.hackaday.io/v1' HAD_AUTH_URI='https://hackaday.io/authorize'\ '?client_id=%(CLIENT_ID)s'\ '&response_type=code' HAD_TOKEN_URI='https://auth.hackaday.io/access_token'\ '?client_id=%(CLIENT_ID)s'\ '&client_secret=%(CLIENT_SECRET)s'\ '&code=%(CODE)s'\ '&grant_type=authorization_code' # Rate limiting RQLIM_TIME=30 # seconds def __init__(self, client_id, client_secret, api_key, api_uri=HAD_API_URI, auth_uri=HAD_AUTH_URI, token_uri=HAD_TOKEN_URI, rqlim_time=RQLIM_TIME, client=None, log=None, io_loop=None): if log is None: log = extdlog.getLogger(self.__class__.__module__) if io_loop is None: io_loop = IOLoop.current() if client is None: client = AsyncHTTPClient() self._client = client self._io_loop = io_loop self._log = log self._client_id = client_id self._client_secret = client_secret self._api_key = api_key self._api_uri = api_uri self._auth_uri = auth_uri self._token_uri = token_uri # Timestamps of last rqlim_num requests self._last_rq = 0.0 self._rqlim_time = rqlim_time # Semaphore to limit concurrent access self._rq_sem = Semaphore(1) # If None, then no "forbidden" status is current. # Otherwise, this stores when the "forbidden" flag expires. self._forbidden_expiry = None @property def is_forbidden(self): """ Return true if the last request returned a "forbidden" response code and was made within the last hour. """ if self._forbidden_expiry is None: return False return self._forbidden_expiry > self._io_loop.time() @coroutine def _ratelimit_sleep(self): """ Ensure we don't exceed the rate limit by tracking the request timestamps and adding a sleep if required. """ now = self._io_loop.time() # Figure out if we need to wait before the next request delay = (self._last_rq + self._rqlim_time) - now self._log.trace('Last request at %f, delay: %f', self._last_rq, delay) if delay <= 0: # Nope, we're clear return self._log.debug('Waiting %f sec for rate limit', delay) yield sleep(delay) self._log.trace('Resuming operations') def _decode(self, response, default_encoding='UTF-8'): """ Decode a given reponse body. """ return decode_body(response.headers['Content-Type'], response.body, default_encoding) @coroutine def api_fetch(self, uri, **kwargs): """ Make a raw request whilst respecting the HAD API request limits. This is primarily to support retrieval of avatars and other data without hitting the HAD.io site needlessly hard. """ if 'connect_timeout' not in kwargs: kwargs['connect_timeout'] = 120.0 if 'request_timeout' not in kwargs: kwargs['request_timeout'] = 120.0 try: yield self._rq_sem.acquire() while True: try: yield self._ratelimit_sleep() response = yield self._client.fetch(uri, **kwargs) self._last_rq = self._io_loop.time() self._log.audit('Request:\n' '%s %s\n' 'Headers: %s\n' 'Response: %s\n' 'Headers: %s\n' 'Body:\n%s', response.request.method, response.request.url, response.request.headers, response.code, response.headers, response_text(response)) break except gaierror as e: if e.errno != EAGAIN: raise raise except HTTPError as e: if e.response is not None: self._log.audit('Request:\n' '%s %s\n' 'Headers: %s\n' 'Response: %s\n' 'Headers: %s\n' 'Body:\n%s', e.response.request.method, e.response.request.url, e.response.request.headers, e.response.code, e.response.headers, response_text(e.response)) if e.code == 403: # Back-end is rate limiting us. Back off an hour. self._forbidden_expiry = self._io_loop.time() \ + 3600.0 raise except ConnectionResetError: # Back-end is blocking us. Back off 15 minutes. self._forbidden_expiry = self._io_loop.time() \ + 900.0 raise finally: self._rq_sem.release() raise Return(response) @coroutine def _api_call(self, uri, query=None, token=None, api_key=True, **kwargs): headers = kwargs.setdefault('headers', {}) headers.setdefault('Accept', 'application/json') if token is not None: headers['Authorization'] = 'token %s' % token if query is None: query = {} if api_key: query.setdefault('api_key', self._api_key) self._log.audit('Query arguments: %r', query) encode_kv = lambda k, v : '%s=%s' % (k, urlparse.quote_plus(str(v))) def encode_item(item): (key, value) = item if isinstance(value, list): return '&'.join(map(lambda v : encode_kv(key, v), value)) else: return encode_kv(key, value) if len(query) > 0: uri += '?%s' % '&'.join(map(encode_item, query.items())) if not uri.startswith('http'): uri = self._api_uri + uri self._log.audit('%s %r', kwargs.get('method','GET'), uri) response = yield self.api_fetch(uri, **kwargs) # If we get here, then our service is back. self._forbidden_expiry = None (ct, ctopts, body) = self._decode(response) if ct.lower() != 'application/json': raise ValueError('Server returned unrecognised type %s' % ct) raise Return(json.loads(body)) # oAuth endpoints @property def auth_uri(self): """ Return the auth URI that we need to send the user to if they're not logged in. """ return self._auth_uri % dict(CLIENT_ID=self._client_id) def get_token(self, code): """ Fetch the token for API queries from the authorization code given. """ # Determine where to retrieve the access token from post_uri = self._token_uri % dict( CLIENT_ID=urlparse.quote_plus(self._client_id), CLIENT_SECRET=urlparse.quote_plus(self._client_secret), CODE=urlparse.quote_plus(code) ) return self._api_call( post_uri, method='POST', body=b'', api_key=False) # Pagination options def _page_query_opts(self, page, per_page): query = {} if page is not None: query['page'] = int(page) if per_page is not None: query['per_page'] = int(per_page) return query # User API endpoints def get_current_user(self, token): """ Fetch the current user's profile information. """ return self._api_call('/me', token=token) def _user_query_opts(self, sortby, page, per_page): query = self._page_query_opts(page, per_page) sortby = UserSortBy(sortby) query['sortby'] = sortby.value return query _GET_USERS_WORKAROUND_RE = re.compile( r'<a href="([^"]+)" class="hacker-image">') _PRIVATE_MSG_LINK_RE = re.compile( r'<a href="/messages/new\?user=(\d+)">') @coroutine def get_user_ids(self, sortby=UserSortBy.influence, page=None): if page is None: page = 1 sortby = UserSortBy(sortby) response = yield self.api_fetch( 'https://hackaday.io/hackers?sort=%s&page=%d' \ % (sortby.value, page)) (ct, ctopts, body) = self._decode(response) # Body is in HTML, look for links to profile pages pages = [] for line in body.split('\n'): match = self._GET_USERS_WORKAROUND_RE.search(line) if match: pages.append(match.group(1)) ids = [] # Fetch each profile page (ugh!) and look for user ID # This is literally all we need at this point, the rest we'll # get from the API. for page in pages: if page.startswith('/'): page = 'https://hackaday.io' + page response = yield self.api_fetch(page) (ct, ctopts, body) = self._decode(response) for line in body.split('\n'): match = self._PRIVATE_MSG_LINK_RE.search(line) if match: ids.append(int(match.group(1))) break raise Return(ids) @coroutine def _get_users_workaround(self, sortby=UserSortBy.influence, page=None): ids = yield self.get_user_ids(sortby, page) users = yield self.get_users(ids=ids) raise Return(users) @coroutine def get_users(self, sortby=UserSortBy.influence, ids=None, page=None, per_page=None): """ Retrieve a list of all users """ query = self._user_query_opts(sortby, page, per_page) if ids is None: # sortby==newest is broken, has been for a while now. if sortby == UserSortBy.newest: result = yield self._get_users_workaround( sortby, query.get('page')) else: result = yield self._api_call('/users', query=query) elif isinstance(ids, slice): query['ids'] = '%d,%d' % (ids.start, ids.stop) result = yield self._api_call('/users/range', query=query) else: ids = set(ids) if len(ids) > 50: raise ValueError('Too many IDs') query['ids'] = ','.join(['%d' % uid for uid in ids]) result = yield self._api_call('/users/batch', query=query) raise Return(result) def search_users(self, screen_name=None, location=None, tag=None, sortby=UserSortBy.influence, page=None, per_page=None): query = self._user_query_opts(sortby, page, per_page) for (arg, val) in ( ('screen_name', screen_name), ('location', location), ('tag', tag) ): if val is not None: query[arg] = str(val) return self._api_call('/users/search', query=query) def get_user(self, user_id): return self._api_call('/users/%d' % user_id) def get_user_followers(self, user_id, sortby=UserSortBy.influence, page=None, per_page=None): query = self._user_query_opts(sortby, page, per_page) return self._api_call('/users/%d/followers' % user_id, query=query) def get_user_following(self, user_id, sortby=UserSortBy.influence, page=None, per_page=None): query = self._user_query_opts(sortby, page, per_page) return self._api_call('/users/%d/following' % user_id, query=query) def get_user_projects(self, user_id, sortby=ProjectSortBy.skulls, page=None, per_page=None): query = self._project_query_opts(sortby, page, per_page) return self._api_call('/users/%d/projects' % user_id, query=query) def get_user_skulls(self, user_id, sortby=UserSortBy.influence, page=None, per_page=None): query = self._user_query_opts(sortby, page, per_page) return self._api_call('/users/%d/skulls' % user_id, query=query) def get_user_links(self, user_id, page=None, per_page=None): query = self._page_query_opts(page, per_page) return self._api_call('/users/%d/links' % user_id, query=query) def get_user_tags(self, user_id, page=None, per_page=None): query = self._page_query_opts(page, per_page) return self._api_call('/users/%d/tags' % user_id, query=query) def get_user_pages(self, user_id, page=None, per_page=None): query = self._page_query_opts(page, per_page) return self._api_call('/users/%d/pages' % user_id, query=query) # Projects API def _project_query_opts(self, sortby, page, per_page): query = self._page_query_opts(page, per_page) sortby = ProjectSortBy(sortby) query['sortby'] = sortby.value return query def get_projects(self, sortby=ProjectSortBy.skulls, ids=None, page=None, per_page=None): """ Retrieve a list of all projects """ query = self._project_query_opts(sortby, page, per_page) if ids is None: return self._api_call('/projects', query=query) elif isinstance(ids, slice): query['ids'] = '%d,%d' % (slice.start, slice.stop) return self._api_call('/projects/range', query=query) else: ids = set(ids) if len(ids) > 50: raise ValueError('Too many IDs') query['ids'] = ','.join(['%d' % pid for pid in ids]) return self._api_call('/projects/batch', query=query) def search_projects(self, term, sortby=ProjectSortBy.skulls, page=None, per_page=None): query = self._project_query_opts(sortby, page, per_page) query['search_term'] = str(term) return self._api_call('/projects/search', query=query) def get_project(self, project_id): return self._api_call('/projects/%d' % project_id) def get_project_team(self, project_id, sortby=UserSortBy.influence, page=None, per_page=None): query = self._user_query_opts(sortby, page, per_page) return self._api_call('/projects/%d/team' % project_id, query=query) def get_project_followers(self, project_id, sortby=UserSortBy.influence, page=None, per_page=None): query = self._user_query_opts(sortby, page, per_page) return self._api_call('/projects/%d/followers' % project_id, query=query) def get_project_skulls(self, project_id, sortby=UserSortBy.influence, page=None, per_page=None): query = self._user_query_opts(sortby, page, per_page) return self._api_call('/projects/%d/skulls' % project_id, query=query) def get_project_comments(self, project_id, sortby=UserSortBy.influence, page=None, per_page=None): query = self._user_query_opts(sortby, page, per_page) return self._api_call('/projects/%d/comments' % project_id, query=query) def get_project_links(self, project_id, page=None, per_page=None): query = self._page_query_opts(page, per_page) return self._api_call('/projects/%d/links' % project_id, query=query) def get_project_images(self, project_id, page=None, per_page=None): query = self._page_query_opts(page, per_page) return self._api_call('/projects/%d/images' % project_id, query=query) def get_project_components(self, project_id, page=None, per_page=None): query = self._page_query_opts(page, per_page) return self._api_call('/projects/%d/components' % project_id, query=query) def get_project_tags(self, project_id, page=None, per_page=None): query = self._page_query_opts(page, per_page) return self._api_call('/projects/%d/tags' % project_id, query=query) def get_project_logs(self, project_id, page=None, per_page=None): query = self._page_query_opts(page, per_page) return self._api_call('/projects/%d/logs' % project_id, query=query) def get_project_instructions(self, project_id, page=None, per_page=None): query = self._page_query_opts(page, per_page) return self._api_call('/projects/%d/instructions' % project_id, query=query) def get_project_details(self, project_id, page=None, per_page=None): query = self._page_query_opts(page, per_page) return self._api_call('/projects/%d/details' % project_id, query=query)
import threading, psycopg2 as pg2 import datetime from tornado.locks import Semaphore from tornado import gen from collections import OrderedDict trunks = OrderedDict() trunks_groups = OrderedDict() trunks_semaphore = Semaphore(1) @gen.coroutine def _read_trunks(): global trunks global trunks_groups if trunks and trunks_groups: raise gen.Return(trunks) trunks = OrderedDict() trunks_groups = OrderedDict() try: db = pg2.connect( 'host=192.168.222.20 port=5432 dbname=freeswitch_trunks user=freeswitch password=freeswitch' ) c = db.cursor() SELECT = 'select tr.name as group_trunks, trl.phone as trunk_name, o.name as operator_name, o.icon_name, ' \ 'trl.in_trunk_position, c.id as channel_id, trl.direction, d.id as device_id, ' \ 'c.lines as max_lines, c.sip_gateway_name, d.address, c.port, dcl.name as device_class, tr.screen_position, ' \ 'trl.trunk_id ' \ 'from trunk_lines trl ' \ 'left join operators o on (o.id=trl.operator_id) ' \ 'left join trunks tr on (tr.id=trl.trunk_id) ' \
def cull_idle( url, api_token, profiles_list = [], db_filename = "profile_quotas.db", check_every = 600, concurrency=10 ): """Shutdown idle single-user servers""" auth_header = {'Authorization': 'token %s' % api_token} req = HTTPRequest(url=url + '/users', headers=auth_header) now = datetime.now(timezone.utc) client = AsyncHTTPClient() if concurrency: semaphore = Semaphore(concurrency) @coroutine def fetch(req): """client.fetch wrapped in a semaphore to limit concurrency""" yield semaphore.acquire() try: return (yield client.fetch(req)) finally: yield semaphore.release() else: fetch = client.fetch resp = yield fetch(req) users = json.loads(resp.body.decode('utf8', 'replace')) futures = [] @coroutine def handle_server(user, server_name, server): """Handle (maybe) culling a single server "server" is the entire server model from the API. Returns True if server is now stopped (user removable), False otherwise. """ log_name = user['name'] if server_name: log_name = '%s/%s' % (user['name'], server_name) if server.get('pending'): app_log.warning( "Not culling server %s with pending %s", log_name, server['pending'] ) return False # jupyterhub < 0.9 defined 'server.url' once the server was ready # as an *implicit* signal that the server was ready. # 0.9 adds a dedicated, explicit 'ready' field. # By current (0.9) definitions, servers that have no pending # events and are not ready shouldn't be in the model, # but let's check just to be safe. if not server.get('ready', bool(server['url'])): app_log.warning( "Not culling not-ready not-pending server %s: %s", log_name, server ) return False if server.get('started'): age = now - parse_date(server['started']) else: # started may be undefined on jupyterhub < 0.9 age = None # CUSTOM CULLING TEST CODE HERE # Add in additional server tests here. Return False to mean "don't # cull", True means "cull immediately", or, for example, update some # other variables like inactive_limit. # # Here, server['state'] is the result of the get_state method # on the spawner. This does *not* contain the below by # default, you may have to modify your spawner to make this # work. The `user` variable is the user model from the API. # # if server['state']['profile_name'] == 'unlimited' # return False # inactive_limit = server['state']['culltime'] should_cull = False # if there's no profile info in the server state to base the determinaton on, we got nothing to go on profile_slug = server.get("state", {}).get("profile_slug", None) balance = float("inf") if profile_slug: conn = db.get_connection(db_filename) db.update_user_tokens(conn, profiles_list, user['name'], user['admin']) for profile in profiles_list: if profile["slug"] == profile_slug and "quota" in profile: hours = (check_every / 60 / 60) db.log_usage(conn, profiles_list, user['name'], profile_slug, hours, user['admin']) db.charge_tokens(conn, profiles_list, user['name'], profile_slug, hours, user['admin']) current_balance = db.get_balance(conn, profiles_list, user['name'], profile_slug, user['admin']) if current_balance < 0.0: pass # don't actually cull, let the balance go negative (since we don't have a way to alert the user that their server is about to be killed) # should_cull = True db.close_connection(conn) if should_cull: app_log.info( "Culling server %s (balance for profile %s is %s)", log_name, profile_slug, balance ) if not should_cull: app_log.debug( "Not culling server %s (balance for profile %s is %s)", log_name, profile_slug, balance, ) return False if server_name: # culling a named server delete_url = url + "/users/%s/servers/%s" % ( quote(user['name']), quote(server['name']), ) else: delete_url = url + '/users/%s/server' % quote(user['name']) req = HTTPRequest(url=delete_url, method='DELETE', headers=auth_header) resp = yield fetch(req) if resp.code == 202: app_log.warning("Server %s is slow to stop", log_name) # return False to prevent culling user with pending shutdowns return False return True @coroutine def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. # Hub doesn't allow deleting users with running servers. # jupyterhub 0.9 always provides a 'servers' model. # 0.8 only does this when named servers are enabled. if 'servers' in user: servers = user['servers'] else: # jupyterhub < 0.9 without named servers enabled. # create servers dict with one entry for the default server # from the user model. # only if the server is running. servers = {} if user['server']: servers[''] = { 'last_activity': user['last_activity'], 'pending': user['pending'], 'url': user['server'], } server_futures = [ handle_server(user, server_name, server) for server_name, server in servers.items() ] results = yield multi(server_futures) for user in users: futures.append((user['name'], handle_user(user))) for (name, f) in futures: try: result = yield f except Exception: app_log.exception("Error processing %s", name) else: if result: app_log.debug("Finished culling %s", name)
class BaseHandler(tornado.web.RequestHandler): pairs = {} # type: Dict[str, str] analyzers = {} # type: Dict[str, Tuple[str, str]] generators = {} # type: Dict[str, Tuple[str, str]] taggers = {} # type: Dict[str, Tuple[str, str]] spellers = {} # type: Dict[str, Tuple[str, str]] # (l1, l2): [translation.Pipeline], only contains flushing pairs! pipelines = {} # type: Dict[str, List] pipelines_holding = [] # type: List callback = None timeout = None scale_mt_logs = False verbosity = 0 api_keys_conf = None # dict representing a graph of translation pairs; keys are source languages # e.g. pairs_graph['eng'] = ['fra', 'spa'] pairs_graph = {} # type: Dict[str, List[str]] # 2-D dict storing the shortest path for a chained translation pair # keys are source and target languages # e.g. paths['eng']['fra'] = ['eng', 'spa', 'fra'] paths = {} # type: Dict[str, Dict[str, List[str]]] stats = { 'startdate': datetime.now(), 'useCount': {}, 'vmsize': 0, 'timing': [], } # (l1, l2): translation.ParsedModes pipeline_cmds = {} # type: Dict max_pipes_per_pair = 1 min_pipes_per_pair = 0 max_users_per_pipe = 5 max_idle_secs = 0 restart_pipe_after = 1000 doc_pipe_sem = Semaphore(3) # Empty the url_cache[pair] when it's this full: max_inmemory_url_cache = 1000 # type: int url_cache = {} # type: Dict[Tuple[str, str], Dict[str, str]] url_cache_path = None # type: Optional[str] # Keep half a gig free when storing url_cache to disk: min_free_space_disk_url_cache = 512 * 1024 * 1024 # type: int def initialize(self): self.callback = self.get_argument('callback', default=None) @classmethod def init_pairs_graph(cls): for pair in cls.pairs: lang1, lang2 = pair.split('-') if lang1 in cls.pairs_graph: cls.pairs_graph[lang1].append(lang2) else: cls.pairs_graph[lang1] = [lang2] @classmethod def calculate_paths(cls, start): nodes = set() for pair in map(lambda x: x.split('-'), cls.pairs): nodes.add(pair[0]) nodes.add(pair[1]) dists = {} prevs = {} dists[start] = 0 while nodes: u = min(nodes, key=lambda u: dists.get(u, sys.maxsize)) nodes.remove(u) for v in cls.pairs_graph.get(u, []): if v in nodes: other = dists.get( u, sys.maxsize ) + 1 # TODO: weight(u, v) -- lower weight = better translation if other < dists.get(v, sys.maxsize): dists[v] = other prevs[v] = u cls.paths[start] = {} for u in prevs: prev = prevs[u] path = [u] while prev: path.append(prev) prev = prevs.get(prev) cls.paths[start][u] = list(reversed(path)) @classmethod def init_paths(cls): for lang in cls.pairs_graph: cls.calculate_paths(lang) def log_vmsize(self): if self.verbosity < 1: return scale = {'kB': 1024, 'mB': 1048576, 'KB': 1024, 'MB': 1048576} try: for line in open('/proc/%d/status' % os.getpid()): if line.startswith('VmSize:'): _, num, unit = line.split() break vmsize = int(num) * scale[unit] if vmsize > self.stats['vmsize']: logging.warning('VmSize of %s from %d to %d', os.getpid(), self.stats['vmsize'], vmsize) self.stats['vmsize'] = vmsize except Exception as e: # Don't fail just because we couldn't log: logging.info('Exception in log_vmsize: %s', e) def send_response(self, data): self.log_vmsize() if isinstance(data, dict) or isinstance(data, list): data = dump_json(data) self.set_header('Content-Type', 'application/json; charset=UTF-8') if self.callback: self.set_header('Content-Type', 'application/javascript; charset=UTF-8') self._write_buffer.append(utf8('%s(%s)' % (self.callback, data))) else: self._write_buffer.append(utf8(data)) self.finish() def write_error(self, status_code, **kwargs): http_explanations = { 400: 'Request not properly formatted or contains languages that Apertium APy does not support', 404: 'Resource requested does not exist. URL may have been mistyped', 408: 'Server did not receive a complete request within the time it was prepared to wait. Try again', 500: 'Unexpected condition on server. Request could not be fulfilled.', } explanation = kwargs.get('explanation', http_explanations.get(status_code, '')) if 'exc_info' in kwargs and len(kwargs['exc_info']) > 1: exception = kwargs['exc_info'][1] if hasattr(exception, 'log_message') and exception.log_message: explanation = exception.log_message % exception.args elif hasattr(exception, 'reason'): explanation = exception.reason or tornado.httputil.responses.get( status_code, 'Unknown') else: explanation = tornado.httputil.responses.get( status_code, 'Unknown') result = { 'status': 'error', 'code': status_code, 'message': tornado.httputil.responses.get(status_code, 'Unknown'), 'explanation': explanation, } data = dump_json(result) self.set_header('Content-Type', 'application/json; charset=UTF-8') if self.callback: self.set_header('Content-Type', 'application/javascript; charset=UTF-8') self._write_buffer.append(utf8('%s(%s)' % (self.callback, data))) else: self._write_buffer.append(utf8(data)) self.finish() def set_default_headers(self): self.set_header('Access-Control-Allow-Origin', '*') self.set_header('Access-Control-Allow-Methods', 'GET,POST,OPTIONS') self.set_header( 'Access-Control-Allow-Headers', 'accept, cache-control, origin, x-requested-with, x-file-name, content-type' ) @tornado.web.asynchronous def post(self): self.get() def options(self): self.set_status(204) self.finish()
class TornadoTransmission(): def __init__(self, max_concurrent_batches=10, block_on_send=False, block_on_response=False, max_batch_size=100, send_frequency=timedelta(seconds=0.25), user_agent_addition=''): if not has_tornado: raise ImportError( 'TornadoTransmission requires tornado, but it was not found.' ) self.block_on_send = block_on_send self.block_on_response = block_on_response self.max_batch_size = max_batch_size self.send_frequency = send_frequency user_agent = "libhoney-py/" + VERSION if user_agent_addition: user_agent += " " + user_agent_addition self.http_client = AsyncHTTPClient( force_instance=True, defaults=dict(user_agent=user_agent)) # libhoney adds events to the pending queue for us to send self.pending = Queue(maxsize=1000) # we hand back responses from the API on the responses queue self.responses = Queue(maxsize=2000) self.batch_data = {} self.sd = statsd.StatsClient(prefix="libhoney") self.batch_sem = Semaphore(max_concurrent_batches) def start(self): ioloop.IOLoop.current().spawn_callback(self._sender) def send(self, ev): '''send accepts an event and queues it to be sent''' self.sd.gauge("queue_length", self.pending.qsize()) try: if self.block_on_send: self.pending.put(ev) else: self.pending.put_nowait(ev) self.sd.incr("messages_queued") except QueueFull: response = { "status_code": 0, "duration": 0, "metadata": ev.metadata, "body": "", "error": "event dropped; queue overflow", } if self.block_on_response: self.responses.put(response) else: try: self.responses.put_nowait(response) except QueueFull: # if the response queue is full when trying to add an event # queue is full response, just skip it. pass self.sd.incr("queue_overflow") # We're using the older decorator/yield model for compatibility with # Python versions before 3.5. # See: http://www.tornadoweb.org/en/stable/guide/coroutines.html#python-3-5-async-and-await @gen.coroutine def _sender(self): '''_sender is the control loop that pulls events off the `self.pending` queue and submits batches for actual sending. ''' events = [] last_flush = time.time() while True: try: ev = yield self.pending.get(timeout=self.send_frequency) if ev is None: # signals shutdown yield self._flush(events) return events.append(ev) if (len(events) > self.max_batch_size or time.time() - last_flush > self.send_frequency.total_seconds()): yield self._flush(events) events = [] except TimeoutError: yield self._flush(events) events = [] last_flush = time.time() @gen.coroutine def _flush(self, events): if not events: return for dest, group in group_events_by_destination(events).items(): yield self._send_batch(dest, group) @gen.coroutine def _send_batch(self, destination, events): ''' Makes a single batch API request with the given list of events. The `destination` argument contains the write key, API host and dataset name used to build the request.''' start = time.time() status_code = 0 try: # enforce max_concurrent_batches yield self.batch_sem.acquire() url = urljoin(urljoin(destination.api_host, "/1/batch/"), destination.dataset) payload = [] for ev in events: event_time = ev.created_at.isoformat() if ev.created_at.tzinfo is None: event_time += "Z" payload.append({ "time": event_time, "samplerate": ev.sample_rate, "data": ev.fields() }) req = HTTPRequest( url, method='POST', headers={ "X-Honeycomb-Team": destination.writekey, "Content-Type": "application/json", }, body=json.dumps(payload, default=json_default_handler), ) self.http_client.fetch(req, self._response_callback) # store the events that were sent so we can process responses later # it is important that we delete these eventually, or we'll run into memory issues self.batch_data[req] = {"start": start, "events": events} except Exception as e: # Catch all exceptions and hand them to the responses queue. self._enqueue_errors(status_code, e, start, events) finally: self.batch_sem.release() def _enqueue_errors(self, status_code, error, start, events): for ev in events: self.sd.incr("send_errors") self._enqueue_response(status_code, "", error, start, ev.metadata) def _response_callback(self, resp): # resp.request should be the same HTTPRequest object built by _send_batch # and mapped to values in batch_data events = self.batch_data[resp.request]["events"] start = self.batch_data[resp.request]["start"] try: status_code = resp.code resp.rethrow() statuses = [d["status"] for d in json.loads(resp.body)] for ev, status in zip(events, statuses): self._enqueue_response(status, "", None, start, ev.metadata) self.sd.incr("messages_sent") except Exception as e: self._enqueue_errors(status_code, e, start, events) self.sd.incr("send_errors") finally: # clean up the data for this batch del self.batch_data[resp.request] def _enqueue_response(self, status_code, body, error, start, metadata): resp = { "status_code": status_code, "body": body, "error": error, "duration": (time.time() - start) * 1000, "metadata": metadata } if self.block_on_response: self.responses.put(resp) else: try: self.responses.put_nowait(resp) except QueueFull: pass def close(self): '''call close to send all in-flight requests and shut down the senders nicely. Times out after max 20 seconds per sending thread plus 10 seconds for the response queue''' try: self.pending.put(None, 10) except QueueFull: pass # signal to the responses queue that nothing more is coming. try: self.responses.put(None, 10) except QueueFull: pass def get_response_queue(self): ''' return the responses queue on to which will be sent the response objects from each event send''' return self.responses
class ManagedKernelPool(KernelPool): ''' Spawns a pool of kernels. Manages access to individual kernels using a borrower/lender pattern. Cleans them all up when shut down. ''' def __init__(self, prespawn_count, kernel_manager): # Make sure there's at least one kernel as a delegate if not prespawn_count: prespawn_count = 1 super(ManagedKernelPool, self).__init__(prespawn_count, kernel_manager) self.kernel_clients = {} self.on_recv_funcs = {} self.pool_index = 0 self.kernel_pool = [] kernel_ids = self.kernel_manager.list_kernel_ids() self.kernel_semaphore = Semaphore(len(kernel_ids)) # Connect to any prespawned kernels for kernel_id in kernel_ids: self.kernel_clients[kernel_id] = kernel_manager.get_kernel( kernel_id).client() self.kernel_pool.append(kernel_id) iopub = self.kernel_manager.connect_iopub(kernel_id) iopub.on_recv(self.create_on_reply(kernel_id)) @gen.coroutine def acquire(self): ''' Returns a kernel client and id for use and removes the kernel the resource pool. Kernels must be returned via the release method. :return: Returns a kernel client and a kernel id ''' yield self.kernel_semaphore.acquire() kernel_id = self.kernel_pool[0] del self.kernel_pool[0] raise gen.Return((self.kernel_clients[kernel_id], kernel_id)) def release(self, kernel_id): ''' Returns a kernel back to the resource pool. :param kernel_id: Id of the kernel to return to the pool ''' self.kernel_pool.append(kernel_id) self.kernel_semaphore.release() def _on_reply(self, kernel_id, msg_list): idents, msg_list = self.kernel_clients[ kernel_id].session.feed_identities(msg_list) msg = self.kernel_clients[kernel_id].session.deserialize(msg_list) self.on_recv_funcs[kernel_id](msg) def create_on_reply(self, kernel_id): ''' The lambda is used to handle a specific reply per kernel and provide a unique stack scope per invocation. ''' return lambda msg_list: self._on_reply(kernel_id, msg_list) def on_recv(self, kernel_id, func): ''' Registers a callback for io_pub messages for a particular kernel. This is needed to avoid having multiple callbacks per kernel client. :param kernel_id: Id of the kernel :param func: Callback function to handle the message ''' self.on_recv_funcs[kernel_id] = func def shutdown(self): ''' Shuts down all kernels in the pool and in the kernel manager. ''' for kid in self.kernel_clients: self.kernel_clients[kid].stop_channels() self.kernel_manager.shutdown_kernel(kid, now=True) # Any remaining kernels that were not created for our pool should be shutdown super(ManagedKernelPool, self).shutdown()
import misaka import psutil import requests from feedgen.feed import FeedGenerator from pytube import YouTube from tornado import gen, httputil, ioloop, iostream, process, web from tornado.locks import Semaphore __version__ = '3.0' key = None video_links = {} playlist_feed = {} channel_feed = {} conversion_queue = {} converting_lock = Semaphore(2) def get_youtube_url(video): if video in video_links and video_links[video][ 'expire'] > datetime.datetime.now(): return video_links[video]['url'] yt = YouTube('http://www.youtube.com/watch?v=' + video) vid = yt.streams.get_highest_resolution().url parts = { part.split('=')[0]: part.split('=')[1] for part in vid.split('?')[-1].split('&') } link = { 'url': vid, 'expire': datetime.datetime.fromtimestamp(int(parts['expire']))
class ManagedKernelPool(KernelPool): """Spawns a pool of kernels that are treated as identical delegates for future requests. Manages access to individual kernels using a borrower/lender pattern. Cleans them all up when shut down. Parameters ---------- prespawn_count Number of kernels to spawn immediately kernel_manager Kernel manager instance Attributes ---------- kernel_clients : dict Map of kernel IDs to client instances for communicating with them on_recv_funcs : dict Map of kernel IDs to iopub callback functions kernel_pool : list List of available delegate kernel IDs kernel_semaphore : tornado.locks.Semaphore Semaphore that controls access to the kernel pool """ def __init__(self, prespawn_count, kernel_manager): # Make sure there's at least one kernel as a delegate if not prespawn_count: prespawn_count = 1 super(ManagedKernelPool, self).__init__(prespawn_count, kernel_manager) self.kernel_clients = {} self.on_recv_funcs = {} self.kernel_pool = [] kernel_ids = self.kernel_manager.list_kernel_ids() self.kernel_semaphore = Semaphore(len(kernel_ids)) # Create clients and iopub handlers for prespawned kernels for kernel_id in kernel_ids: self.kernel_clients[kernel_id] = kernel_manager.get_kernel(kernel_id).client() self.kernel_pool.append(kernel_id) iopub = self.kernel_manager.connect_iopub(kernel_id) iopub.on_recv(self.create_on_reply(kernel_id)) @gen.coroutine def acquire(self): """Gets a kernel client and removes it from the available pool of clients. Returns ------- tuple Kernel client instance, kernel ID """ yield self.kernel_semaphore.acquire() kernel_id = self.kernel_pool[0] del self.kernel_pool[0] raise gen.Return((self.kernel_clients[kernel_id], kernel_id)) def release(self, kernel_id): """Puts a kernel back into the pool of kernels available to handle requests. Parameters ---------- kernel_id : str Kernel to return to the pool """ self.kernel_pool.append(kernel_id) self.kernel_semaphore.release() def _on_reply(self, kernel_id, msg_list): """Invokes the iopub callback registered for the `kernel_id` and passes it a deserialized list of kernel messsages. Parameters ---------- kernel_id : str Kernel that sent the reply msg_list : list List of 0mq messages """ idents, msg_list = self.kernel_clients[kernel_id].session.feed_identities(msg_list) msg = self.kernel_clients[kernel_id].session.deserialize(msg_list) self.on_recv_funcs[kernel_id](msg) def create_on_reply(self, kernel_id): """Creates an anonymous function to handle reply messages from the kernel. Parameters ---------- kernel_id Kernel to listen to Returns ------- function Callback function taking a kernel ID and 0mq message list """ return lambda msg_list: self._on_reply(kernel_id, msg_list) def on_recv(self, kernel_id, func): """Registers a callback function for iopub messages from a particular kernel. This is needed to avoid having multiple callbacks per kernel client. Parameters ---------- kernel_id Kernel from which to receive iopub messages func Callback function to use for kernel iopub messages """ self.on_recv_funcs[kernel_id] = func def shutdown(self): """Shuts down all kernels and their clients. """ for kid in self.kernel_clients: self.kernel_clients[kid].stop_channels() self.kernel_manager.shutdown_kernel(kid, now=True) # Any remaining kernels that were not created for our pool should be shutdown super(ManagedKernelPool, self).shutdown()
class Crawler(object): def _init_defaults(self): self.start_link = None self.link_priority = 2 self.img_priority = 8 self.politeness = 2 self.workers_limit = 10 # allow at most 10 concurrent workers self.link_regex = re.compile("^http://.*") self.img_regex = re.compile(".*") self.fname_digits = 4 self.min_width = 200 self.min_height = 200 self.img_dir = "E:/tmp/" self.idle_wait_loops = 100 self.port = 8888 def _load_config(self): parser = ConfigParser.ConfigParser() parser.read("config.ini") if parser.has_option("global", "starturl"): starturl = parser.get("global", "starturl") self.start_link = starturl if parser.has_option("global", "linkregex"): self.link_regex = re.compile(parser.get("global", "linkregex")) if parser.has_option("global", "imgregex"): self.img_regex = re.compile(parser.get("global", "imgregex")) if parser.has_option("global", "politeness"): politeness = parser.getint("global", "politeness") if politeness <= 0: print "politeness must be a positive integer" raise SystemExit() self.politeness = politeness if parser.has_option("global", "imgdir"): imgdir = parser.get("global", "imgdir") if not os.path.exists(imgdir) or not os.path.isdir(imgdir): print "invalid imgdir configuration" raise SystemExit() if not imgdir.endswith("/"): imgdir += "/" self.img_dir = imgdir if parser.has_option("global", "minwidth"): width = parser.getint("global", "minwidth") self.min_width = width if parser.has_option("global", "minheight"): height = parser.getint("global", "minheight") self.min_height = height def __init__(self, start_link=None): self._init_defaults() # Now load the config file to override defaults self._load_config() if start_link: self.start_link = start_link if not self.start_link: raise SystemExit("No start link is provided, exiting now...") links.put(self.start_link) self.semaphore = Semaphore(self.workers_limit) @gen.coroutine def run(self): # First start an debug server app = Application([(r"/", WebHandler)]) server = HTTPServer(app) server.listen(self.port) idle_loops = 0 while True: if imageurls.qsize() == 0 and links.qsize() == 0: print "Both link and image queues are empty now" idle_loops += 1 if idle_loops == self.idle_wait_loops: break else: idle_loops = 0 # clear the idle loop counter if imageurls.qsize() == 0: self.handle_links() elif links.qsize() == 0: self.handle_imageurls() else: choices = [0] * self.link_priority + [1] * self.img_priority choice = random.choice(choices) if choice: self.handle_imageurls() else: self.handle_links() yield gen.sleep(0.1 * self.politeness) # Wait for all link handlers links.join() # Handling imageurls if generated by the last few links while imageurls.qsize(): self.handle_imageurls() imageurls.join() @gen.coroutine def handle_links(self): yield self.semaphore.acquire() newlink = yield links.get() # Make sure we haven't visited this one if newlink in visited_links: self.semaphore.release() raise gen.Return() visited_links.add(newlink) # use async client to fetch this url client = AsyncHTTPClient() tries = 3 # Give it 3 chances before putting it in failure while tries: response = yield client.fetch(newlink) if response.code == 200: break tries -= 1 # release the semaphore self.semaphore.release() if response.code != 200: link_failures.append(newlink) print "[FAILURE] - %s" % newlink raise gen.Return() # TODO: replace this with a report api print "[VISITED] - %s" % newlink # parse url to get the base url components = urlparse.urlparse(newlink) baseurl = components[0] + "://" + components[1] path = components[2] # parse the html with bs soup = bs4.BeautifulSoup(response.body) # extract valid links and put into links a_tags = soup.find_all("a") for tag in a_tags: if "href" not in tag.attrs: continue href = tag["href"] if href.startswith("#"): continue if href.startswith("/"): # relative href = baseurl + href else: if not path.endswith("/"): path = path[: path.rfind("/") + 1] href = baseurl + "/" + path + href if not self.link_regex.match(href): continue if href in visited_links: continue links.put(href) print "NEWLINK:", href # extract imgs and put into imageurls img_tags = soup.find_all("img") for tag in img_tags: if "src" not in tag.attrs: continue src = tag["src"] if src.startswith("/"): # relative src = baseurl + src if not self.img_regex.match(src): continue if src in downloaded_images: continue imageurls.put(src) print "NEW IMAGE:", src # now the task is done links.task_done() @gen.coroutine def handle_imageurls(self): yield self.semaphore.acquire() imgurl = yield imageurls.get() if imgurl in downloaded_images: self.semaphore.release() raise gen.Return() # mark the image as downloaded downloaded_images.add(imgurl) # use async client to fetch this url client = AsyncHTTPClient() tries = 3 # Give it 3 chances before putting it in failure while tries: response = yield client.fetch(imgurl) if response.code == 200: break tries -= 1 # Download is finished, release semaphore self.semaphore.release() if response.code != 200: download_failures.append(imgurl) print "[FAILURE] - %s" % imgurl raise gen.Return() # TODO: replace this with a report api print "[DOWNLOADED] - %s" % imgurl # Read the file content img = PIL.Image.open(response.buffer) w, h = img.size if w < self.min_width or h < self.min_height: raise gen.Return() # find out the image extension, default to jpg if "." in imgurl: ext = imgurl.split(".")[-1].lower() if ext not in ["jpg", "png", "gif"]: ext = "jpg" elif img.format: ext = img.format.lower() else: ext = "jpg" # increment the counter global img_counter img_counter += 1 fname = str(img_counter).zfill(self.fname_digits) + "." + ext fpath = self.img_dir + fname # save the image file f = open(fpath, "wb") f.write(response.body) # now the task is done imageurls.task_done()
def _read_trunks(): global trunks global trunks_groups if trunks and trunks_groups: raise gen.Return(trunks) trunks = OrderedDict() trunks_groups = OrderedDict() try: db = pg2.connect( 'host=192.168.222.20 port=5432 dbname=freeswitch_trunks user=freeswitch password=freeswitch' ) c = db.cursor() SELECT = 'select tr.name as group_trunks, trl.phone as trunk_name, o.name as operator_name, o.icon_name, ' \ 'trl.in_trunk_position, c.id as channel_id, trl.direction, d.id as device_id, ' \ 'c.lines as max_lines, c.sip_gateway_name, d.address, c.port, dcl.name as device_class, tr.screen_position, ' \ 'trl.trunk_id ' \ 'from trunk_lines trl ' \ 'left join operators o on (o.id=trl.operator_id) ' \ 'left join trunks tr on (tr.id=trl.trunk_id) ' \ 'left join channels c on (c.id=trl.channel_id) ' \ 'left join devices d on (d.id=c.device_id) ' \ 'left join device_classes dcl on (dcl.id=d.class_id) ' \ 'where (trl.channel_id > 0) and (c.is_active) ' \ 'order by tr.screen_position, group_trunks, trl.in_trunk_position ' c.execute(SELECT) for r in c.fetchall(): trunk_name = r[9] if r[ 9] else '0' + r[1] if r[1][:2] != '23' else r[1] # trunk_name = r[9] if r[9] else '0'+r[1] if not trunk_name in trunks: trunks[trunk_name] = OrderedDict() if not r[0] in trunks_groups: trunks_groups[r[0]] = [] trunks_groups[r[0]].append(trunk_name) trunks[trunk_name].update( callers=dict(), # Обрабатываемы(е/й) звонок(и) max_lines=r[8], # channels.lines operator_logo='static/img/%s.png' % r[3], # operators.icon_name channel_id=r[5], # channels.id counters={ 'answered': 0, 'total': 0, 'rejected': 0 }, # Счётчики direction=r[ 6], # trunk_lines.directions ['inbound', 'outbound', 'sms'] channel='%s:%s' % (r[10], r[11]), # device.address, channel.port group=r[0], # trunks.name semaphore=Semaphore(1), # Блокировка транка phone=r[1], # Номер телефона на линии device_id=r[7], screen_position=r[13], trunk_id=r[14], ) print(trunk_name) c.close() db.close() except Exception as e: print('ChannelHandler._read_trunks исключение: %s' % e) # print ('ChannelHandler._read_trunks данные: %s' % trunks) raise gen.Return(trunks)
def cull_idle(url, api_token, inactive_limit, cull_users=False, max_age=0, concurrency=10): """Shutdown idle single-user servers If cull_users, inactive *users* will be deleted as well. """ auth_header = { 'Authorization': 'token %s' % api_token, } req = HTTPRequest( url=url + '/users', headers=auth_header, ) now = datetime.now(timezone.utc) client = AsyncHTTPClient() if concurrency: semaphore = Semaphore(concurrency) @coroutine def fetch(req): """client.fetch wrapped in a semaphore to limit concurrency""" yield semaphore.acquire() try: return (yield client.fetch(req)) finally: yield semaphore.release() else: fetch = client.fetch resp = yield fetch(req) users = json.loads(resp.body.decode('utf8', 'replace')) futures = [] @coroutine def handle_server(user, server_name, server): """Handle (maybe) culling a single server Returns True if server was culled, False otherwise. """ log_name = user['name'] if server_name: log_name = '%s/%s' % (user['name'], server_name) if server.get('pending'): app_log.warning("Not culling server %s with pending %s", log_name, server['pending']) return False if server.get('started'): age = now - parse_date(server['started']) else: # started may be undefined on jupyterhub < 0.9 age = 'unknown' # check last activity # last_activity can be None in 0.9 if server['last_activity']: inactive = now - parse_date(server['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'started' field which is never None # for running servers inactive = age should_cull = inactive.total_seconds() >= inactive_limit if should_cull: app_log.info("Culling server %s (inactive for %s)", log_name, format_td(inactive)) if max_age and not should_cull: # only check started if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age.total_seconds() >= max_age: app_log.info("Culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive)) should_cull = True if not should_cull: app_log.debug("Not culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive)) return False req = HTTPRequest( url=url + '/users/%s/server' % quote(user['name']), method='DELETE', headers=auth_header, ) resp = yield fetch(req) if resp.code == 202: app_log.warning( "Server %s is slow to stop", log_name, ) # return False to prevent culling user with pending shutdowns return False return True @coroutine def handle_user(user): """Handle one user""" # shutdown servers first. # Hub doesn't allow deleting users with running servers. servers = user.get( 'servers', { '': { 'started': user.get('started'), 'last_activity': user['last_activity'], 'pending': user['pending'], } }) server_futures = [ handle_server(user, server_name, server) for server_name, server in servers.items() ] results = yield multi(server_futures) if not cull_users: return # some servers are still running, cannot cull users still_alive = len(results) - sum(results) if still_alive: app_log.debug("Not culling user %s with %i servers still alive", user['name'], still_alive) return False should_cull = False if user.get('created'): age = now - parse_date(user['created']) else: # created may be undefined on jupyterhub < 0.9 age = 'unknown' # check last activity # last_activity can be None in 0.9 if user['last_activity']: inactive = now - parse_date(user['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'created' field which is never None inactive = age should_cull = inactive.total_seconds() >= inactive_limit if should_cull: app_log.info("Culling user %s (inactive for %s)", user['name'], inactive) if max_age and not should_cull: # only check created if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age.total_seconds() >= max_age: app_log.info("Culling user %s (age: %s, inactive for %s)", user['name'], format_td(age), format_td(inactive)) should_cull = True if not should_cull: app_log.debug("Not culling user %s (created: %s, last active: %s)", user['name'], format_td(age), format_td(inactive)) return False req = HTTPRequest( url=url + '/users/%s' % user['name'], method='DELETE', headers=auth_header, ) yield fetch(req) return True for user in users: futures.append((user['name'], handle_user(user))) for (name, f) in futures: try: result = yield f except Exception: app_log.exception("Error processing %s", name) else: if result: app_log.debug("Finished culling %s", name)
class IPCMessageSubscriber(IPCClient): ''' Salt IPC message subscriber Create an IPC client to receive messages from IPC publisher An example of a very simple IPCMessageSubscriber connecting to an IPCMessagePublisher. This example assumes an already running IPCMessagePublisher. IMPORTANT: The below example also assumes the IOLoop is NOT running. # Import Tornado libs import tornado.ioloop # Import Salt libs import salt.config import salt.transport.ipc # Create a new IO Loop. # We know that this new IO Loop is not currently running. io_loop = tornado.ioloop.IOLoop() ipc_publisher_socket_path = '/var/run/ipc_publisher.ipc' ipc_subscriber = salt.transport.ipc.IPCMessageSubscriber(ipc_server_socket_path, io_loop=io_loop) # Connect to the server # Use the associated IO Loop that isn't running. io_loop.run_sync(ipc_subscriber.connect) # Wait for some data package = ipc_subscriber.read_sync() ''' def __singleton_init__(self, socket_path, io_loop=None): super(IPCMessageSubscriber, self).__singleton_init__(socket_path, io_loop=io_loop) self._read_sync_future = None self._read_stream_future = None self._sync_ioloop_running = False self.saved_data = [] self._sync_read_in_progress = Semaphore() self.callbacks = set() self.reading = False @tornado.gen.coroutine def _read_sync(self, timeout): yield self._sync_read_in_progress.acquire() exc_to_raise = None ret = None try: while True: if self._read_stream_future is None: self._read_stream_future = self.stream.read_bytes( 4096, partial=True) if timeout is None: wire_bytes = yield self._read_stream_future else: future_with_timeout = FutureWithTimeout( self.io_loop, self._read_stream_future, timeout) wire_bytes = yield future_with_timeout self._read_stream_future = None # Remove the timeout once we get some data or an exception # occurs. We will assume that the rest of the data is already # there or is coming soon if an exception doesn't occur. timeout = None self.unpacker.feed(wire_bytes) first = True for framed_msg in self.unpacker: if first: ret = framed_msg['body'] first = False else: self.saved_data.append(framed_msg['body']) if not first: # We read at least one piece of data break except TornadoTimeoutError: # In the timeout case, just return None. # Keep 'self._read_stream_future' alive. ret = None except tornado.iostream.StreamClosedError as exc: log.trace('Subscriber disconnected from IPC %s', self.socket_path) self._read_stream_future = None exc_to_raise = exc except Exception as exc: log.error( 'Exception occurred in Subscriber while handling stream: %s', exc) self._read_stream_future = None exc_to_raise = exc if self._sync_ioloop_running: # Stop the IO Loop so that self.io_loop.start() will return in # read_sync(). self.io_loop.spawn_callback(self.io_loop.stop) if exc_to_raise is not None: raise exc_to_raise # pylint: disable=E0702 self._sync_read_in_progress.release() raise tornado.gen.Return(ret) def read_sync(self, timeout=None): ''' Read a message from an IPC socket The socket must already be connected. The associated IO Loop must NOT be running. :param int timeout: Timeout when receiving message :return: message data if successful. None if timed out. Will raise an exception for all other error conditions. ''' if self.saved_data: return self.saved_data.pop(0) self._sync_ioloop_running = True self._read_sync_future = self._read_sync(timeout) self.io_loop.start() self._sync_ioloop_running = False ret_future = self._read_sync_future self._read_sync_future = None return ret_future.result() @tornado.gen.coroutine def _read_async(self, callback): while not self.stream.closed(): try: self._read_stream_future = self.stream.read_bytes(4096, partial=True) self.reading = True wire_bytes = yield self._read_stream_future self._read_stream_future = None self.unpacker.feed(wire_bytes) for framed_msg in self.unpacker: body = framed_msg['body'] self.io_loop.spawn_callback(callback, body) except tornado.iostream.StreamClosedError: log.trace('Subscriber disconnected from IPC %s', self.socket_path) break except Exception as exc: log.error( 'Exception occurred while Subscriber handling stream: %s', exc) yield tornado.gen.sleep(1) def __run_callbacks(self, raw): for callback in self.callbacks: self.io_loop.spawn_callback(callback, raw) @tornado.gen.coroutine def read_async(self): ''' Asynchronously read messages and invoke a callback when they are ready. :param callback: A callback with the received data ''' while not self.connected(): try: yield self.connect(timeout=5) except tornado.iostream.StreamClosedError: log.trace('Subscriber closed stream on IPC %s before connect', self.socket_path) yield tornado.gen.sleep(1) except Exception as exc: log.error('Exception occurred while Subscriber connecting: %s', exc) yield tornado.gen.sleep(1) yield self._read_async(self.__run_callbacks) def close(self): ''' Routines to handle any cleanup before the instance shuts down. Sockets and filehandles should be closed explicitly, to prevent leaks. ''' if not self._closing: IPCClient.close(self) if self._closing: # This will prevent this message from showing up: # '[ERROR ] Future exception was never retrieved: # StreamClosedError' if self._read_sync_future is not None and self._read_sync_future.done( ): self._read_sync_future.exception() if self._read_stream_future is not None and self._read_stream_future.done( ): self._read_stream_future.exception()
def cull_idle(url, api_token, inactive_limit, cull_users=False, remove_named_servers=False, max_age=0, concurrency=10): """Shutdown idle single-user servers If cull_users, inactive *users* will be deleted as well. """ auth_header = {'Authorization': 'token %s' % api_token} req = HTTPRequest(url=url + '/users', headers=auth_header) now = datetime.now(timezone.utc) client = AsyncHTTPClient() if concurrency: semaphore = Semaphore(concurrency) @coroutine def fetch(req): """client.fetch wrapped in a semaphore to limit concurrency""" yield semaphore.acquire() try: return (yield client.fetch(req)) finally: yield semaphore.release() else: fetch = client.fetch resp = yield fetch(req) users = json.loads(resp.body.decode('utf8', 'replace')) futures = [] @coroutine def handle_server(user, server_name, server, max_age, inactive_limit): """Handle (maybe) culling a single server "server" is the entire server model from the API. Returns True if server is now stopped (user removable), False otherwise. """ log_name = user['name'] if server_name: log_name = '%s/%s' % (user['name'], server_name) if server.get('pending'): app_log.warning("Not culling server %s with pending %s", log_name, server['pending']) return False # jupyterhub < 0.9 defined 'server.url' once the server was ready # as an *implicit* signal that the server was ready. # 0.9 adds a dedicated, explicit 'ready' field. # By current (0.9) definitions, servers that have no pending # events and are not ready shouldn't be in the model, # but let's check just to be safe. if not server.get('ready', bool(server['url'])): app_log.warning("Not culling not-ready not-pending server %s: %s", log_name, server) return False if server.get('started'): age = now - parse_date(server['started']) else: # started may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if server['last_activity']: inactive = now - parse_date(server['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'started' field which is never None # for running servers inactive = age # CUSTOM CULLING TEST CODE HERE # Add in additional server tests here. Return False to mean "don't # cull", True means "cull immediately", or, for example, update some # other variables like inactive_limit. # # Here, server['state'] is the result of the get_state method # on the spawner. This does *not* contain the below by # default, you may have to modify your spawner to make this # work. The `user` variable is the user model from the API. # # if server['state']['profile_name'] == 'unlimited' # return False # inactive_limit = server['state']['culltime'] state = server['state'] # Support getting state from wrapspawer child's conf. if 'child_conf' in state: state = state['child_conf'] if 'cull_max_age' in state: max_age = max(max_age, state['cull_max_age']) if 'cull_inactive_limit' in state: inactive_limit = max(inactive_limit, state['cull_inactive_limit']) app_log.info( f"CULL IDLE: {user['name']}/{server_name}: {max_age} inactive={inactive} inactive_limit={inactive_limit} age={age} last_activity={server['last_activity']}" ) should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling server %s (inactive for %s)", log_name, format_td(inactive)) if max_age and not should_cull: # only check started if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive), ) should_cull = True if not should_cull: app_log.debug( "Not culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive), ) return False body = None if server_name: # culling a named server # A named server can be stopped and kept available to the user # for starting again or stopped and removed. To remove the named # server we have to pass an additional option in the body of our # DELETE request. delete_url = url + "/users/%s/servers/%s" % ( quote(user['name']), quote(server['name']), ) if remove_named_servers: body = json.dumps({"remove": True}) else: delete_url = url + '/users/%s/server' % quote(user['name']) req = HTTPRequest(url=delete_url, method='DELETE', headers=auth_header, body=body, allow_nonstandard_methods=True) resp = yield fetch(req) if resp.code == 202: app_log.warning("Server %s is slow to stop", log_name) # return False to prevent culling user with pending shutdowns return False return True @coroutine def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. # Hub doesn't allow deleting users with running servers. # jupyterhub 0.9 always provides a 'servers' model. # 0.8 only does this when named servers are enabled. if 'servers' in user: servers = user['servers'] else: # jupyterhub < 0.9 without named servers enabled. # create servers dict with one entry for the default server # from the user model. # only if the server is running. servers = {} if user['server']: servers[''] = { 'last_activity': user['last_activity'], 'pending': user['pending'], 'url': user['server'], } server_futures = [ handle_server(user, server_name, server, max_age, inactive_limit) for server_name, server in servers.items() ] results = yield multi(server_futures) if not cull_users: return # some servers are still running, cannot cull users still_alive = len(results) - sum(results) if still_alive: app_log.debug( "Not culling user %s with %i servers still alive", user['name'], still_alive, ) return False should_cull = False if user.get('created'): age = now - parse_date(user['created']) else: # created may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if user['last_activity']: inactive = now - parse_date(user['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'created' field which is never None inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling user %s (inactive for %s)", user['name'], inactive) if max_age and not should_cull: # only check created if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling user %s (age: %s, inactive for %s)", user['name'], format_td(age), format_td(inactive), ) should_cull = True if not should_cull: app_log.debug( "Not culling user %s (created: %s, last active: %s)", user['name'], format_td(age), format_td(inactive), ) return False req = HTTPRequest(url=url + '/users/%s' % user['name'], method='DELETE', headers=auth_header) yield fetch(req) return True for user in users: futures.append((user['name'], handle_user(user))) for (name, f) in futures: try: result = yield f except Exception: app_log.exception("Error processing %s", name) else: if result: app_log.debug("Finished culling %s", name)
def __init__(self, method='ascii', **kwargs): super(AsyncModbusGeneratorClient, self).__init__(method=method, **kwargs) self.sem = Semaphore(1)
async def cull_idle(api_url, base_url, api_token, inactive_limit, max_age=0, warn_timeout=0, concurrency=10, verify_ssl=True): """Shutdown idle single-user servers If cull_users, inactive *users* will be deleted as well. """ auth_header = { 'Authorization': 'token %s' % api_token, } req = HTTPRequest( url=api_url + '/users', headers=auth_header, validate_cert=verify_ssl, ) now = datetime.now(timezone.utc) client = AsyncHTTPClient() if concurrency: semaphore = Semaphore(concurrency) async def fetch(req): """client.fetch wrapped in a semaphore to limit concurrency""" await semaphore.acquire() try: return (await client.fetch(req)) finally: semaphore.release() else: fetch = client.fetch # tornado.curl_httpclient.CurlError: HTTP 599: Connection timed out after 20003 milliseconds # Potential timeout error here? (slow to stop line: 478) resp = await fetch(req) users = json.loads(resp.body.decode('utf8', 'replace')) futures = [] async def get_server_active(server): server_url = urljoin(base_url, server['url']) app_log.debug('Server url: %s', server_url) if server.get('started'): age = now - parse_date(server['started']) else: # started may be undefined on jupyterhub < 0.9 age = None # check server status num_kernels = 0 try: # status query does not change last_activity on notebook server req = HTTPRequest( url=urljoin(server_url, 'api/status'), headers=auth_header, validate_cert=verify_ssl, ) resp = await fetch(req) status = json.loads(resp.body.decode('utf-8', 'replace')) # app_log.info(status) inactive = [now - parse_date(status['last_activity'])] num_kernels = status['kernels'] except HTTPClientError as e: app_log.error('Failed to get notebook status: %s', e) # make sure inactive is defined inactive = [age] # if an error happened, then num_kernels is still 0 # TODO: for now kernel activity tracking is deactivated # code below is problematic ... it triggers an update of last activity on # the notebook server ... also should look into other activites like open shell (process?) # a busy cell that finishes updates last_activity as well # Also it seems, that a user has to keep the notebook in an open tab visible/foreground .... #. putting tab a side does not help.not #. minifing browser window neither or moving off screen neither. # hiding browser window with anothe window stops refreshing as well #. jupyterlab stops polling if document.hidden is true (old interface doesn't poll at all) #. -> we could also hook into here ... and add a 'keep-alive' extension, that keeps polling (at a slower interval or so?) # TODO: to make this more reliable, we should install a notebook api/service extension, #. that tracks all the activity we want. This allows us to use the internal # notebook API and container/host process inspection to look at more things as well if not num_kernels: # no kernel running return True, min(inactive), age # FIXME: hardcoded skip rest of activity checking return True, min(inactive), age # assume everything is idle idle = True # kernels: # TODO: we ar ecalling through the proxy here.... which will update # the hubs view of inactivity :( if app_log.isEnabledFor(logging.DEBUG): app_log.debug('Query kernels %s', urljoin(server_url, 'api/kernels')) req = HTTPRequest( url=urljoin(server_url, 'api/kernels'), headers=auth_header, validate_cert=verify_ssl, ) try: resp = await fetch(req) kernels = json.loads(resp.body.decode('utf-8', 'replace')) for kernel in kernels: # TODO: seems like kernel state stays in 'starting' after a restart and auto # re-creation of running kernels from last ui state idle = idle and (kernel['execution_state'] in ('idle', 'starting')) inactive.append(now - parse_date(kernel['last_activity'])) except HTTPClientError as e: app_log.error('Falid to inspect notebook kernels: %s', e) # find smallest inactive time return idle, min(inactive), age async def handle_server(user, server_name, server): """Handle (maybe) culling a single server Returns True if server is now stopped (user removable), False otherwise. """ # import ipdb; ipdb.set_trace() log_name = user['name'] if server_name: log_name = '%s/%s' % (user['name'], server_name) if server.get('pending'): app_log.warning("Not culling server %s with pending %s", log_name, server['pending']) return False # jupyterhub < 0.9 defined 'server.url' once the server was ready # as an *implicit* signal that the server was ready. # 0.9 adds a dedicated, explicit 'ready' field. # By current (0.9) definitions, servers that have no pending # events and are not ready shouldn't be in the model, # but let's check just to be safe. if not server.get('ready', bool(server['url'])): app_log.warning("Not culling not-ready not-pending server %s: %s", log_name, server) return False idle, inactive, age = await get_server_active(server) if not idle and app_log.isEnabledFor(logging.DEBUG): # something is not idle # when the kernel transitions from busy to idle, the kernel resets the # inactive timer as well. app_log.debug( 'Not culling server %s with busy connections. (inactive for %s)', log_name, inactive) return should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling server %s (inactive for %s)", log_name, format_td(inactive)) if max_age and not should_cull: # only check started if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info("Culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive)) should_cull = True # should we warn? remaining = inactive_limit - inactive.total_seconds() if should_warn(user, warn_timeout, inactive) and remaining > 0: IOLoop.current().run_in_executor( None, send_email, user, { 'serverurl': urljoin(base_url, server['url']), 'inactive': human_seconds(inactive.total_seconds()), 'remaining': human_seconds(remaining), }) if not should_cull: app_log.debug("Not culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive)) return False if server_name: # culling a named server delete_url = api_url + "/users/%s/servers/%s" % (quote( user['name']), quote(server['name'])) else: delete_url = api_url + '/users/%s/server' % quote(user['name']) req = HTTPRequest( url=delete_url, method='DELETE', headers=auth_header, validate_cert=verify_ssl, ) resp = await fetch(req) if resp.code == 202: app_log.warning( "Server %s is slow to stop", log_name, ) # return False to prevent culling user with pending shutdowns return False return True async def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. servers = user['servers'] server_futures = [ handle_server(user, server_name, server) for server_name, server in servers.items() ] results = await multi(server_futures) for user in users: futures.append((user['name'], handle_user(user))) for (name, f) in futures: try: result = await f except Exception: app_log.exception("Error processing %s", name) else: if result: app_log.debug("Finished culling %s", name)
return int(round(time.time() * 1000)) request_users = {} class User(object): def __init__(self): self.last_visit_time = 0 self.req_num = 0 index = 0 worker_stop = 0 concurrent_worker_count = 20 consumer = Semaphore(concurrent_worker_count) request_queue = Queue(maxsize=5000) async def worker(worker_id): print("worker {} start".format(worker_id)) while not worker_stop: await consumer.acquire() req_handler = await request_queue.get() print("worker {} work".format(worker_id)) try: global index remote_ip = req_handler.request.remote_ip message = "love you liuchen {} {}".format(remote_ip, index) index = index + 1 #process grpc here
class IPCMessageSubscriber(IPCClient): ''' Salt IPC message subscriber Create an IPC client to receive messages from IPC publisher An example of a very simple IPCMessageSubscriber connecting to an IPCMessagePublisher. This example assumes an already running IPCMessagePublisher. IMPORTANT: The below example also assumes the IOLoop is NOT running. # Import Tornado libs import tornado.ioloop # Import Salt libs import salt.config import salt.transport.ipc # Create a new IO Loop. # We know that this new IO Loop is not currently running. io_loop = tornado.ioloop.IOLoop() ipc_publisher_socket_path = '/var/run/ipc_publisher.ipc' ipc_subscriber = salt.transport.ipc.IPCMessageSubscriber(ipc_server_socket_path, io_loop=io_loop) # Connect to the server # Use the associated IO Loop that isn't running. io_loop.run_sync(ipc_subscriber.connect) # Wait for some data package = ipc_subscriber.read_sync() ''' def __singleton_init__(self, socket_path, io_loop=None): super(IPCMessageSubscriber, self).__singleton_init__( socket_path, io_loop=io_loop) self._read_sync_future = None self._read_stream_future = None self._sync_ioloop_running = False self.saved_data = [] self._sync_read_in_progress = Semaphore() @tornado.gen.coroutine def _read_sync(self, timeout): yield self._sync_read_in_progress.acquire() exc_to_raise = None ret = None try: while True: if self._read_stream_future is None: self._read_stream_future = self.stream.read_bytes(4096, partial=True) if timeout is None: wire_bytes = yield self._read_stream_future else: future_with_timeout = FutureWithTimeout( self.io_loop, self._read_stream_future, timeout) wire_bytes = yield future_with_timeout self._read_stream_future = None # Remove the timeout once we get some data or an exception # occurs. We will assume that the rest of the data is already # there or is coming soon if an exception doesn't occur. timeout = None self.unpacker.feed(wire_bytes) first = True for framed_msg in self.unpacker: if first: ret = framed_msg['body'] first = False else: self.saved_data.append(framed_msg['body']) if not first: # We read at least one piece of data break except tornado.ioloop.TimeoutError: # In the timeout case, just return None. # Keep 'self._read_stream_future' alive. ret = None except tornado.iostream.StreamClosedError as exc: log.trace('Subscriber disconnected from IPC {0}'.format(self.socket_path)) self._read_stream_future = None exc_to_raise = exc except Exception as exc: log.error('Exception occurred in Subscriber while handling stream: {0}'.format(exc)) self._read_stream_future = None exc_to_raise = exc if self._sync_ioloop_running: # Stop the IO Loop so that self.io_loop.start() will return in # read_sync(). self.io_loop.spawn_callback(self.io_loop.stop) if exc_to_raise is not None: raise exc_to_raise # pylint: disable=E0702 self._sync_read_in_progress.release() raise tornado.gen.Return(ret) def read_sync(self, timeout=None): ''' Read a message from an IPC socket The socket must already be connected. The associated IO Loop must NOT be running. :param int timeout: Timeout when receiving message :return: message data if successful. None if timed out. Will raise an exception for all other error conditions. ''' if self.saved_data: return self.saved_data.pop(0) self._sync_ioloop_running = True self._read_sync_future = self._read_sync(timeout) self.io_loop.start() self._sync_ioloop_running = False ret_future = self._read_sync_future self._read_sync_future = None return ret_future.result() @tornado.gen.coroutine def _read_async(self, callback): while not self.stream.closed(): try: self._read_stream_future = self.stream.read_bytes(4096, partial=True) wire_bytes = yield self._read_stream_future self._read_stream_future = None self.unpacker.feed(wire_bytes) for framed_msg in self.unpacker: body = framed_msg['body'] self.io_loop.spawn_callback(callback, body) except tornado.iostream.StreamClosedError: log.trace('Subscriber disconnected from IPC {0}'.format(self.socket_path)) break except Exception as exc: log.error('Exception occurred while Subscriber handling stream: {0}'.format(exc)) @tornado.gen.coroutine def read_async(self, callback): ''' Asynchronously read messages and invoke a callback when they are ready. :param callback: A callback with the received data ''' while not self.connected(): try: yield self.connect(timeout=5) except tornado.iostream.StreamClosedError: log.trace('Subscriber closed stream on IPC {0} before connect'.format(self.socket_path)) yield tornado.gen.sleep(1) except Exception as exc: log.error('Exception occurred while Subscriber connecting: {0}'.format(exc)) yield tornado.gen.sleep(1) yield self._read_async(callback) def close(self): ''' Routines to handle any cleanup before the instance shuts down. Sockets and filehandles should be closed explicitly, to prevent leaks. ''' if not self._closing: IPCClient.close(self) # This will prevent this message from showing up: # '[ERROR ] Future exception was never retrieved: # StreamClosedError' if self._read_sync_future is not None: self._read_sync_future.exc_info() if self._read_stream_future is not None: self._read_stream_future.exc_info() def __del__(self): if IPCMessageSubscriber in globals(): self.close()