class FrameSaver( threading.Thread ): def __init__( self ): threading.Thread.__init__( self ) self.daemon = True self.name = 'FrameSaver' self.reset() def reset( self ): self.queue = Queue() def run( self ): self.reset() while 1: message = self.queue.get() if message[0] == 'Save': cmd, fileName, bib, t, frame = message #sys.stderr.write( 'save' ) PhotoFinish.SavePhoto( fileName, bib, t, frame ) self.queue.task_done() elif message[0] == 'Terminate': self.queue.task_done() self.reset() break def stop( self ): self.queue.put( ['Terminate'] ) self.join() def save( self, fileName, bib, t, frame ): self.queue.put( ['Save', fileName, bib, t, frame] )
class TagGroup( object ): ''' Process groups of tag reads and return the best time estimated using quadratic regression. Stray reads are also detected if there is no quiet period for the tag. The first read time of each stray read is returned. ''' def __init__( self ): self.q = Queue() self.tagInfo = {} def add( self, antenna, tag, t, db ): self.q.put((antenna, tag, t, db)) def flush( self ): # Process all waiting reads. while 1: try: antenna, tag, t, db = self.q.get(False) except Empty: break try: self.tagInfo[tag].add( antenna, t, db ) except KeyError: self.tagInfo[tag] = TagGroupEntry( antenna, t, db ) self.q.task_done() def getReadsStrays( self, tNow=None, method=QuadraticRegressionMethod, antennaChoice=MostReadsChoice, removeOutliers=True ): ''' Returns two lists: reads = [(tag1, t1, sampleSize1, antennaID1), (tag2, t2, sampleSize2, , antennaID2), ...] strays = [(tagA, tFirstReadA), (tagB, tFirstReadB), ...] Each stray will be reported as a read the first time it is detected. ''' self.flush() trNow = datetimeToTr( tNow or datetime.now() ) reads, strays = [], [] toDelete = [] for tag, tge in six.iteritems(self.tagInfo): if trNow - tge.lastReadMax >= tQuiet: # Tag has left read range. if not tge.isStray: t, sampleSize, antennaID = tge.getBestEstimate(method, antennaChoice, removeOutliers) reads.append( (tag, t, sampleSize, antennaID) ) toDelete.append( tag ) elif tge.lastReadMax - tge.firstReadMin >= tStray: # This is a stray. t = trToDatetime( tge.firstReadMin ) if not tge.isStray: tge.setStray() reads.append( (tag, t, 1, 0) ) # Report stray first read time. strays.append( (tag, t) ) for tag in toDelete: del self.tagInfo[tag] reads.sort( key=operator.itemgetter(1,0)) strays.sort( key=operator.itemgetter(1,0) ) return reads, strays
class CountingQueue(object): """A Queue wrapper adding some counting & progress reporting features.""" def __init__(self, **kwargs): # Name is expected to be set after construction, as phases are wired together. self.name = "<unknown queue>" self._get_count = 0 self._put_count = 0 self._done_count = 0 self._lock = Lock() self._delegate = Queue(**kwargs) self.qsize = self._delegate.qsize def _incr_get(self): with self._lock: self._get_count += 1 def _incr_put(self): with self._lock: self._put_count += 1 def _incr_done(self): with self._lock: self._done_count += 1 @property def counts(self): """Returns a QueueCounts tuple for the current state of this queue. Can be used to estimate the progress of a phase reading from this queue. """ with self._lock: return QueueCounts(self._put_count, self._get_count, self._done_count) # The following methods are API-compatible with the standard Queue # methods, but simultaneously update our counts. def put(self, item, *args, **kwargs): self._delegate.put(item, *args, **kwargs) if item not in (Phase.ERROR, Phase.FINISHED): self._incr_put() def get(self, *args, **kwargs): out = self._delegate.get(*args, **kwargs) if out not in (Phase.ERROR, Phase.FINISHED): self._incr_get() return out def task_done(self): self._delegate.task_done() self._incr_done()
class SubscribeListener(SubscribeCallback): def __init__(self): self.connected = False self.connected_event = Event() self.disconnected_event = Event() self.presence_queue = Queue() self.message_queue = Queue() def status(self, pubnub, status): if utils.is_subscribed_event(status) and not self.connected_event.is_set(): self.connected_event.set() elif utils.is_unsubscribed_event(status) and not self.disconnected_event.is_set(): self.disconnected_event.set() def message(self, pubnub, message): self.message_queue.put(message) def presence(self, pubnub, presence): self.presence_queue.put(presence) def wait_for_connect(self): if not self.connected_event.is_set(): self.connected_event.wait() else: raise Exception("the instance is already connected") def wait_for_disconnect(self): if not self.disconnected_event.is_set(): self.disconnected_event.wait() else: raise Exception("the instance is already disconnected") def wait_for_message_on(self, *channel_names): channel_names = list(channel_names) while True: env = self.message_queue.get() self.message_queue.task_done() if env.channel in channel_names: return env else: continue def wait_for_presence_on(self, *channel_names): channel_names = list(channel_names) while True: env = self.presence_queue.get() self.presence_queue.task_done() if env.channel in channel_names: return env else: continue
def _handle_messages_threaded(self): # Handles messages in a threaded fashion. queue = Queue() def producer_loop(): # Read messages from file, and queue them for execution. for msg in self._read_next_message(): queue.put(msg) # Check if an error occurred. if self._done: break # Wait until the queue empties out to signal completion from the # producer's side. if not self._done: queue.join() self._done = True producer = Thread(name="Producer", target=producer_loop) # @note Previously, when trying to do `queue.clear()` in the consumer, # and `queue.join()` in the producer, there would be intermittent # deadlocks. By demoting the producer to a daemon, I (eric.c) have not # yet encountered a deadlock. producer.daemon = True producer.start() # Consume. # TODO(eric.cousineau): Trying to quit via Ctrl+C is awkward (but kinda # works). Is there a way to have `plt.pause` handle Ctrl+C differently? try: pause = self.scope_globals['pause'] while not self._done: # Process messages. while not queue.empty(): msg = queue.get() queue.task_done() self._execute_message(msg) # Spin busy for a bit, let matplotlib (or whatever) flush its # event queue. pause(0.01) except KeyboardInterrupt: # User pressed Ctrl+C. self._done = True print("Quitting") except Exception as e: # We encountered an error, and must stop. self._done = True self._had_error = True traceback.print_exc(file=sys.stderr) sys.stderr.write(" Stopping (--stop_on_error)\n")
def runServer(host='localhost', llrp_host=None, transmitPower=None, receiverSensitivity=None): messageQ = Queue() transmitPower = transmitPower if transmitPower else None receiverSensitivity = receiverSensitivity if receiverSensitivity else None server = None retryDelaySeconds = 3 # Define function to get llrp_host name. if llrp_host and llrp_host.lower() != 'autodetect': LLRPHostFunc = lambda: llrp_host else: LLRPHostFunc = doAutoDetect # Outer loop - connect/reconnect to the reader. while True: if server is not None: writeLog('Attempting reconnect in {} seconds...'.format( retryDelaySeconds)) time.sleep(retryDelaySeconds) server = LLRPServer(LLRPHostFunc=LLRPHostFunc, messageQ=messageQ, transmitPower=transmitPower, receiverSensitivity=receiverSensitivity) writeLog('runServer: LLRP Server on ({}:{})'.format( server.host, server.port)) try: server.connect() writeLog('runServer: Successfully connected to ({}:5084)!'.format( server.llrp_host)) except Exception as e: writeLog('runServer: {}'.format(e)) writeLog('runServer: Connection to ({}:5084) fails.'.format( server.llrp_host)) continue # Inner loop - process messages from the reader. while True: writeLog('runServer: Server: ' + messageQ.get()) messageQ.task_done() if server.exception_termination: writeLog('runServer: Exceptional RFID Reader Termination.') break
class FluentdEvent(object): def __init__(self, app=None): self.app = app if app is not None: self.init_app(app) # Send events after every request finishes app.after_request(self.send_events) # Unbounded queue for sent events self.queue = Queue() def init_app(self, app): tag_prefix = app.config.get("FLUENTD_EVENT_TAG_PREFIX", "flask.fluentd_event") host = app.config.get("FLUENTD_EVENT_HOST", "localhost") port = int(app.config.get("FLUENTD_EVENT_PORT", 24224)) self._sender = sender.FluentSender(tag_prefix, host=host, port=port) # Use the newstyle teardown_appcontext if it's available, # otherwise fall back to the request context if hasattr(app, "teardown_appcontext"): app.teardown_appcontext(self.send_events) else: app.teardown_request(self.send_events) def event(self, tag, event): self.queue.put((tag, event)) def send_events(self, exception): """ Makes a best-effort to send all the events that it pushed during a request but capable of missing some """ pumping = True while pumping: try: tag, event = self.queue.get_nowait() self._sender.emit(tag, event) self.queue.task_done() except Empty: pumping = False except Exception as e: # This is bad but it's worse to foul the request because # of a logging issue logging.exception(e) self.queue.task_done() return exception
class WorkQueue(): def __init__(self, settings_view): self._settings_view = settings_view self._work_queue = Queue() self._worker = threading.Thread(target=self._work_thd) self._worker.daemon = True self._worker.start() def put(self, func, *argv): self._work_queue.put((func, argv)) def _work_thd(self): while True: (func, argv) = self._work_queue.get(block=True) func(*argv) self._work_queue.task_done()
class Fluentd(object): def __init__(self, app=None): self.app = app if app is not None: self.init_app(app) # Send events after every request finishes app.after_request(self.send_events) # Unbounded queue for sent events self.queue = Queue() tag_label = app.config.get('EVENT_TAG_PREFIX', 'flask.fluentd') self._sender = sender.FluentSender(tag_label) def init_app(self, app): # Use the newstyle teardown_appcontext if it's available, # otherwise fall back to the request context if hasattr(app, 'teardown_appcontext'): app.teardown_appcontext(self.send_events) else: app.teardown_request(self.send_events) def event(self, pair): tag, evt = pair self.queue.put((tag, evt)) def send_events(self, exception): """ Makes a best-effort to send all the events that it pushed during a request but capable of missing some """ pumping = True while pumping: try: tag, evt = self.queue.get_nowait() self._sender.emit(tag, evt) self.queue.task_done() except Empty: pumping = False except Exception as e: # This is bad but it's worse to foul the request because # of a logging issue logging.exception(e) self.queue.task_done() return exception
class QueuePublisherMixin(PublisherMixin): queue_publisher_args = { 'interval_start': 2, 'interval_step': 2, 'interval_max': 32, } def __init__(self, **kwargs): super(QueuePublisherMixin, self).__init__(**kwargs) self.__flushing = False self.__fifo = FifoQueue() try: self._register_thread('publisher_queue', self.__run, on_stop=self.__stop) except AttributeError: pass def __run(self, ready_flag, **kwargs): ready_flag.set() publisher_args = self.queue_publisher_args with Connection(self.url, transport_options=publisher_args) as connection: while not self.is_stopping or self.__flushing: try: payload, headers, routing_key = self.__fifo.get() except (Empty, TypeError): self.__flushing = False continue try: with self.Producer(connection, **publisher_args) as publish: publish(payload, headers=headers, routing_key=routing_key) except OperationalError as exc: self.log.error('Publishing queue error: %s', exc, exc_info=1) else: self.__fifo.task_done() def __stop(self): self.__flushing = True self.__fifo.put(None) def publish_soon(self, event, headers=None, routing_key=None, payload=None): headers, payload, routing_key = self._marshal( event, headers, payload, routing_key ) self.__fifo.put((payload, headers, routing_key))
class Events(threading.Thread): def __init__(self, callback): super(Events, self).__init__() self.queue = Queue() # http://stackoverflow.com/a/20598791 self.daemon = False self.callback = callback self.name = "EVENT-QUEUE" self.stop = threading.Event() def put(self, event_type): self.queue.put(event_type) def run(self): """ Actually runs the thread to process events """ try: while not self.stop.is_set(): try: # get event type event_type = self.queue.get(True, 1) # perform callback if we got a event type self.callback(event_type) # event completed self.queue.task_done() except Empty: event_type = None # exiting thread self.stop.clear() except Exception as e: logger.log( u"Exception generated in thread " + self.name + ": " + ex(e), logger.ERROR) logger.log(repr(traceback.format_exc()), logger.DEBUG) # System Events class SystemEvent(Event): RESTART = "RESTART" SHUTDOWN = "SHUTDOWN"
class Events(threading.Thread): def __init__(self, callback): super(Events, self).__init__() self.queue = Queue() # http://stackoverflow.com/a/20598791 self.daemon = False self.callback = callback self.name = 'EVENT-QUEUE' self.stop = threading.Event() def put(self, event_type): self.queue.put(event_type) def run(self): """ Actually runs the thread to process events """ try: while not self.stop.is_set(): try: # get event type event_type = self.queue.get(True, 1) # perform callback if we got a event type self.callback(event_type) # event completed self.queue.task_done() except Empty: event_type = None # exiting thread self.stop.clear() except Exception as error: log.error(u'Exception generated in thread %s: %s', self.name, ex(error)) log.debug(repr(traceback.format_exc())) # System Events class SystemEvent(Event): RESTART = 'RESTART' SHUTDOWN = 'SHUTDOWN'
class Fluentd(object): def __init__(self, app=None): self.app = app if app is not None: self.init_app(app) # Send events after every request finishes app.after_request(self.send_events) # Unbounded queue for sent events self.queue = Queue() tag_label = app.config.get('EVENT_TAG_PREFIX', 'flask.fluentd') self._sender = sender.FluentSender(tag_label) def init_app(self, app): # Use the newstyle teardown_appcontext if it's available, # otherwise fall back to the request context if hasattr(app, 'teardown_appcontext'): app.teardown_appcontext(self.send_events) else: app.teardown_request(self.send_events) def event(self, pair): tag, evt = pair self.queue.put((tag, evt)) def send_events(self, exception): """ Makes a best-effort to send all the events that it pushed during a request but capable of missing some """ pumping = True while pumping: try: tag, evt = self.queue.get_nowait() self._sender.emit(tag, evt) self.queue.task_done() except Empty: pumping = False return exception
class Events(threading.Thread): def __init__(self, callback): super(Events, self).__init__() self.queue = Queue() self.daemon = True self.callback = callback self.name = "EVENT-QUEUE" self.stop = threading.Event() def put(self, type): self.queue.put(type) def run(self): """ Actually runs the thread to process events """ try: while not self.stop.is_set(): try: # get event type type = self.queue.get(True, 1) # perform callback if we got a event type self.callback(type) # event completed self.queue.task_done() except Empty: type = None # exiting thread self.stop.clear() except Exception as e: logger.log("Exception generated in thread " + self.name + ": " + ex(e), logger.ERROR) logger.log(repr(traceback.format_exc()), logger.DEBUG) # System Events class SystemEvent(Event): RESTART = "RESTART" SHUTDOWN = "SHUTDOWN"
class JobWorker(threading.Thread): def __init__(self, cb, sensor_id, result_queue): super(JobWorker, self).__init__() self.cb = cb self.sensor_id = sensor_id self.job_queue = Queue() self.lr_session = None self.result_queue = result_queue def run(self): try: self.lr_session = self.cb.live_response.request_session( self.sensor_id) self.result_queue.put(WorkerStatus(self.sensor_id, status="ready")) while True: work_item = self.job_queue.get(block=True) if not work_item: self.job_queue.task_done() return self.run_job(work_item) self.result_queue.put(CompletionNotification(self.sensor_id)) self.job_queue.task_done() except Exception as e: self.result_queue.put( WorkerStatus(self.sensor_id, status="error", exception=e)) finally: if self.lr_session: self.lr_session.close() self.result_queue.put( WorkerStatus(self.sensor_id, status="exiting")) def run_job(self, work_item): try: work_item.future.set_result(work_item.fn(self.lr_session)) except Exception as e: work_item.future.set_exception(e)
class CrawlerExecutionQueue(object): """Queue and execute operations in a separate thread when celery is running in eager mode. """ def __init__(self): self.queue = Queue() def queue_operation(self, context, data): if not hasattr(self, 'worker'): self.worker = Thread(target=self.execute_crawler) self.worker.setDaemon(True) self.worker.start() self.queue.put((context, data)) def execute_crawler(self): while True: context, data = self.queue.get() context.execute(data) self.queue.task_done() @property def is_empty(self): return not self.queue.unfinished_tasks
def listen(): queue = Queue() def _consume(queue): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.bind(('0.0.0.0', 6899)) print('listening for sdb notifications on :6899...') while True: r, w, x = select.select([sock], [], []) for i in r: data = i.recv(1024) queue.put(data) worker = threading.Thread(target=_consume, args=(queue, )) worker.setDaemon(True) worker.start() orig_tty = termios.tcgetattr(sys.stdin) try: tty.setcbreak(sys.stdin.fileno()) while True: try: port = queue.get(timeout=1) queue.task_done() if port == 'q': break port = int(port) print('opening telnet session at port :%d...' % port) telnet(port).connect() print('listening for sdb notifications on :6899...') except Empty: pass except KeyboardInterrupt: print('got Ctrl-C') queue.put('q') finally: termios.tcsetattr(sys.stdin, termios.TCSADRAIN, orig_tty)
class TagGroup(object): ''' Process groups of tag reads and return the best time estimated using quadratic regression. Stray reads are also detected if there is no quiet period for the tag. The first read time of each stray read is returned. ''' def __init__(self): self.q = Queue() self.tagInfo = {} def add(self, antenna, tag, t, db): self.q.put((antenna, tag, t, db)) def flush(self): # Process all waiting reads. while 1: try: antenna, tag, t, db = self.q.get(False) except Empty: break try: self.tagInfo[tag].add(antenna, t, db) except KeyError: self.tagInfo[tag] = TagGroupEntry(antenna, t, db) self.q.task_done() def getReadsStrays(self, tNow=None, method=QuadraticRegressionMethod, antennaChoice=MostReadsChoice, removeOutliers=True): ''' Returns two lists: reads = [(tag1, t1, sampleSize1, antennaID1), (tag2, t2, sampleSize2, , antennaID2), ...] strays = [(tagA, tFirstReadA), (tagB, tFirstReadB), ...] Each stray will be reported as a read the first time it is detected. ''' self.flush() trNow = datetimeToTr(tNow or datetime.now()) reads, strays = [], [] toDelete = [] for tag, tge in self.tagInfo.items(): if trNow - tge.lastReadMax >= tQuiet: # Tag has left read range. if not tge.isStray: t, sampleSize, antennaID = tge.getBestEstimate( method, antennaChoice, removeOutliers) reads.append((tag, t, sampleSize, antennaID)) toDelete.append(tag) elif tge.lastReadMax - tge.firstReadMin >= tStray: # This is a stray. t = trToDatetime(tge.firstReadMin) if not tge.isStray: tge.setStray() reads.append( (tag, t, 1, 0)) # Report stray first read time. strays.append((tag, t)) for tag in toDelete: del self.tagInfo[tag] reads.sort(key=operator.itemgetter(1, 0)) strays.sort(key=operator.itemgetter(1, 0)) return reads, strays
class TaskIO(object): """Object used to stream I/O between a running Mesos task and the local terminal. :param task: task ID :type task: str :param cmd: a command to launch inside the task's container :type cmd: str :param args: Additional arguments for the command :type args: str :param interactive: whether to attach STDIN of the current terminal to the new command being launched :type interactive: bool :param tty: whether to allocate a tty for this command and attach the local terminal to it :type tty: bool """ # The interval to send heartbeat messages to # keep persistent connections alive. HEARTBEAT_INTERVAL = 30 HEARTBEAT_INTERVAL_NANOSECONDS = HEARTBEAT_INTERVAL * 1000000000 def __init__(self, task_id, cmd=None, args=None, interactive=False, tty=False): # Store relevant parameters of the call for later. self.cmd = cmd self.interactive = interactive self.tty = tty self.args = args # Create a client and grab a reference to the DC/OS master. client = DCOSClient() master = get_master(client) # Get the task and make sure its container was launched by the UCR. # Since task's containers are launched by the UCR by default, we want # to allow most tasks to pass through unchecked. The only exception is # when a task has an explicit container specified and it is not of type # "MESOS". Having a type of "MESOS" implies that it was launched by the # UCR -- all other types imply it was not. task_obj = master.task(task_id) if "container" in task_obj.dict(): if "type" in task_obj.dict()["container"]: if task_obj.dict()["container"]["type"] != "MESOS": raise DCOSException( "This command is only supported for tasks" " launched by the Universal Container Runtime (UCR).") # Get the URL to the agent running the task. if client._mesos_master_url: self.agent_url = client.slave_url( slave_id="", private_url=task_obj.slave().http_url(), path="api/v1") else: self.agent_url = client.slave_url(slave_id=task_obj.slave()['id'], private_url="", path="api/v1") # Grab a reference to the container ID for the task. self.parent_id = master.get_container_id(task_obj) # Generate a new UUID for the nested container # used to run commands passed to `task exec`. self.container_id = str(uuid.uuid4()) # Set up a recordio encoder and decoder # for any incoming and outgoing messages. self.encoder = recordio.Encoder( lambda s: bytes(json.dumps(s, ensure_ascii=False), "UTF-8")) self.decoder = recordio.Decoder( lambda s: json.loads(s.decode("UTF-8"))) # Set up queues to send messages between threads used for # reading/writing to STDIN/STDOUT/STDERR and threads # sending/receiving data over the network. self.input_queue = Queue() self.output_queue = Queue() # Set up an event to block attaching # input until attaching output is complete. self.attach_input_event = threading.Event() self.attach_input_event.clear() # Set up an event to block printing the output # until an attach input event has successfully # been established. self.print_output_event = threading.Event() self.print_output_event.clear() # Set up an event to block the main thread # from exiting until signaled to do so. self.exit_event = threading.Event() self.exit_event.clear() # Use a class variable to store exceptions thrown on # other threads and raise them on the main thread before # exiting. self.exception = None def run(self): """Run the helper threads in this class which enable streaming of STDIN/STDOUT/STDERR between the CLI and the Mesos Agent API. If a tty is requested, we take over the current terminal and put it into raw mode. We make sure to reset the terminal back to its original settings before exiting. """ # Without a TTY. if not self.tty: try: self._start_threads() self.exit_event.wait() except Exception as e: self.exception = e if self.exception: raise self.exception return # With a TTY. if util.is_windows_platform(): raise DCOSException( "Running with the '--tty' flag is not supported on windows.") if not sys.stdin.isatty(): raise DCOSException( "Must be running in a tty to pass the '--tty flag'.") fd = sys.stdin.fileno() oldtermios = termios.tcgetattr(fd) try: if self.interactive: tty.setraw(fd, when=termios.TCSANOW) self._window_resize(signal.SIGWINCH, None) signal.signal(signal.SIGWINCH, self._window_resize) self._start_threads() self.exit_event.wait() except Exception as e: self.exception = e termios.tcsetattr(sys.stdin.fileno(), termios.TCSAFLUSH, oldtermios) if self.exception: raise self.exception def _thread_wrapper(self, func): """A wrapper around all threads used in this class If a thread throws an exception, it will unblock the main thread and save the exception in a class variable. The main thread will then rethrow the exception before exiting. :param func: The start function for the thread :type func: function """ try: func() except Exception as e: self.exception = e self.exit_event.set() def _start_threads(self): """Start all threads associated with this class """ if self.interactive: # Collects input from STDIN and puts # it in the input_queue as data messages. thread = threading.Thread(target=self._thread_wrapper, args=(self._input_thread, )) thread.daemon = True thread.start() # Prepares heartbeat control messages and # puts them in the input queueaat a specific # heartbeat interval. thread = threading.Thread(target=self._thread_wrapper, args=(self._heartbeat_thread, )) thread.daemon = True thread.start() # Opens a persistent connection with the mesos agent and # feeds it both control and data messages from the input # queue via ATTACH_CONTAINER_INPUT messages. thread = threading.Thread(target=self._thread_wrapper, args=(self._attach_container_input, )) thread.daemon = True thread.start() # Opens a persistent connection with a mesos agent, reads # data messages from it and feeds them to an output_queue. thread = threading.Thread( target=self._thread_wrapper, args=(self._launch_nested_container_session, )) thread.daemon = True thread.start() # Collects data messages from the output queue and writes # their content to STDOUT and STDERR. thread = threading.Thread(target=self._thread_wrapper, args=(self._output_thread, )) thread.daemon = True thread.start() def _launch_nested_container_session(self): """Sends a request to the Mesos Agent to launch a new nested container and attach to its output stream. The output stream is then sent back in the response. """ message = { 'type': "LAUNCH_NESTED_CONTAINER_SESSION", 'launch_nested_container_session': { 'container_id': { 'parent': self.parent_id, 'value': self.container_id }, 'command': { 'value': self.cmd, 'arguments': [self.cmd] + self.args, 'shell': False } } } if self.tty: message['launch_nested_container_session']['container'] = { 'type': 'MESOS', 'tty_info': {} } req_extra_args = { 'stream': True, 'headers': { 'Content-Type': 'application/json', 'Accept': 'application/recordio', 'Message-Accept': 'application/json' } } response = http.post(self.agent_url, data=json.dumps(message), timeout=None, **req_extra_args) self._process_output_stream(response) def _process_output_stream(self, response): """Gets data streamed over the given response and places the returned messages into our output_queue. Only expects to receive data messages. :param response: Response from an http post :type response: requests.models.Response """ # Now that we are ready to process the output stream (meaning # our output connection has been established), allow the input # stream to be attached by setting an event. self.attach_input_event.set() # If we are running in interactive mode, wait to make sure that # our input connection succeeds before pushing any output to the # output queue. if self.interactive: self.print_output_event.wait() try: for chunk in response.iter_content(chunk_size=None): records = self.decoder.decode(chunk) for r in records: if r.get('type') and r['type'] == 'DATA': self.output_queue.put(r['data']) except Exception as e: raise DCOSException( "Error parsing output stream: {error}".format(error=e)) self.output_queue.join() self.exit_event.set() def _attach_container_input(self): """Streams all input data (e.g. STDIN) from the client to the agent """ def _initial_input_streamer(): """Generator function yielding the initial ATTACH_CONTAINER_INPUT message for streaming. We have a separate generator for this so that we can attempt the connection once before committing to a persistent connection where we stream the rest of the input. :returns: A RecordIO encoded message """ message = { 'type': 'ATTACH_CONTAINER_INPUT', 'attach_container_input': { 'type': 'CONTAINER_ID', 'container_id': { 'parent': self.parent_id, 'value': self.container_id } } } yield self.encoder.encode(message) def _input_streamer(): """Generator function yielding ATTACH_CONTAINER_INPUT messages for streaming. It yields the _intitial_input_streamer() message, followed by messages from the input_queue on each subsequent call. :returns: A RecordIO encoded message """ yield next(_initial_input_streamer()) while True: record = self.input_queue.get() if not record: break yield record req_extra_args = { 'headers': { 'Content-Type': 'application/recordio', 'Message-Content-Type': 'application/json', 'Accept': 'application/json', 'Connection': 'close', 'Transfer-Encoding': 'chunked' } } # Ensure we don't try to attach our input to a container that isn't # fully up and running by waiting until the # `_process_output_stream` function signals us that it's ready. self.attach_input_event.wait() # Send an intial "Test" message to ensure that we are able to # establish a connection with the agent. If we aren't we will throw # an exception and break out of this thread. However, in cases where # we receive a 500 response from the agent, we actually want to # continue without throwing an exception. A 500 error indicates that # we can't connect to the container because it has already finished # running. In that case we continue running to allow the output queue # to be flushed. try: http.post(self.agent_url, data=_initial_input_streamer(), **req_extra_args) except DCOSHTTPException as e: if not e.response.status_code == 500: raise e # If we succeeded with that connection, unblock process_output_stream() # from sending output data to the output thread. self.print_output_event.set() # Begin streaming the input. http.post(self.agent_url, data=_input_streamer(), timeout=None, **req_extra_args) def _input_thread(self): """Reads from STDIN and places a message with that data onto the input_queue. """ message = { 'type': 'ATTACH_CONTAINER_INPUT', 'attach_container_input': { 'type': 'PROCESS_IO', 'process_io': { 'type': 'DATA', 'data': { 'type': 'STDIN', 'data': '' } } } } for chunk in iter(partial(os.read, sys.stdin.fileno(), 1024), b''): message['attach_container_input']['process_io']['data'][ 'data'] = base64.b64encode(chunk).decode('utf-8') self.input_queue.put(self.encoder.encode(message)) # Push an empty string to indicate EOF to the server and push # 'None' to signal that we are done processing input. message['attach_container_input']['process_io']['data']['data'] = '' self.input_queue.put(self.encoder.encode(message)) self.input_queue.put(None) def _output_thread(self): """Reads from the output_queue and writes the data to the appropriate STDOUT or STDERR. """ while True: # Get a message from the output queue and decode it. # Then write the data to the appropriate stdout or stderr. output = self.output_queue.get() if not output.get('data'): raise DCOSException("Error no 'data' field in output message") data = output['data'] data = base64.b64decode(data.encode('utf-8')) if output.get('type') and output['type'] == 'STDOUT': sys.stdout.buffer.write(data) sys.stdout.flush() elif output.get('type') and output['type'] == 'STDERR': sys.stderr.buffer.write(data) sys.stderr.flush() else: raise DCOSException("Unsupported data type in output stream") self.output_queue.task_done() def _heartbeat_thread(self): """Generates a heartbeat message to send over the ATTACH_CONTAINER_INPUT stream every `interval` seconds and inserts it in the input queue. """ interval = self.HEARTBEAT_INTERVAL nanoseconds = self.HEARTBEAT_INTERVAL_NANOSECONDS message = { 'type': 'ATTACH_CONTAINER_INPUT', 'attach_container_input': { 'type': 'PROCESS_IO', 'process_io': { 'type': 'CONTROL', 'control': { 'type': 'HEARTBEAT', 'heartbeat': { 'interval': { 'nanoseconds': nanoseconds } } } } } } while True: self.input_queue.put(self.encoder.encode(message)) time.sleep(interval) def _window_resize(self, signum, frame): """Signal handler for SIGWINCH. Generates a message with the current demensions of the terminal and puts it in the input_queue. :param signum: the signal number being handled :type signum: int :param frame: current stack frame :type frame: frame """ # Determine the size of our terminal, and create the message to be sent rows, columns = os.popen('stty size', 'r').read().split() message = { 'type': 'ATTACH_CONTAINER_INPUT', 'attach_container_input': { 'type': 'PROCESS_IO', 'process_io': { 'type': 'CONTROL', 'control': { 'type': 'TTY_INFO', 'tty_info': { 'window_size': { 'rows': int(rows), 'columns': int(columns) } } } } } } self.input_queue.put(self.encoder.encode(message))
class CallbackWorkerPool(object): """ A Worker Pool implementation that creates a number of predefined threads used for invoking Session callbacks. """ def __init__(self, write_queue=None, size=1): """ Creates a Callback Worker Pool for use in invoking Session Callbacks when data is received by a push client. :param write_queue: Queue used for queueing up socket write events for when a payload message is received and processed. :param size: The number of worker threads to invoke callbacks. """ # Used to queue up PublishMessageReceived events to be sent back to # the iDigi server. self._write_queue = write_queue # Used to queue up sessions and data to callback with. self._queue = Queue(size) # Number of workers to create. self.size = size self.log = logging.getLogger( '{}.callback_worker_pool'.format(__name__)) for _ in range(size): worker = Thread(target=self._consume_queue) worker.daemon = True worker.start() def _consume_queue(self): """ Continually blocks until data is on the internal queue, then calls the session's registered callback and sends a PublishMessageReceived if callback returned True. """ while True: session, block_id, raw_data = self._queue.get() data = json.loads(raw_data.decode('utf-8')) # decode as JSON try: result = session.callback(data) if result is None: self.log.warn( "Callback %r returned None, expected boolean. Messages " "are not marked as received unless True is returned", session.callback) elif result: # Send a Successful PublishMessageReceived with the # block id sent in request if self._write_queue is not None: response_message = struct.pack( '!HHH', PUBLISH_MESSAGE_RECEIVED, block_id, 200) self._write_queue.put( (session.socket, response_message)) except Exception as exception: self.log.exception(exception) self._queue.task_done() def queue_callback(self, session, block_id, data): """ Queues up a callback event to occur for a session with the given payload data. Will block if the queue is full. :param session: the session with a defined callback function to call. :param block_id: the block_id of the message received. :param data: the data payload of the message received. """ self._queue.put((session, block_id, data))
class LSFBatchSystem(BatchSystemSupport): """ The interface for running jobs on lsf, runs all the jobs you give it as they come in, but in parallel. """ @classmethod def supportsWorkerCleanup(cls): return False @classmethod def supportsHotDeployment(cls): return False def shutdown(self): pass def __init__(self, config, maxCores, maxMemory, maxDisk): super(LSFBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk) self.lsfResultsFile = self._getResultsFileName(config.jobStore) #Reset the job queue and results (initially, we do this again once we've killed the jobs) self.lsfResultsFileHandle = open(self.lsfResultsFile, 'w') self.lsfResultsFileHandle.close( ) #We lose any previous state in this file, and ensure the files existence self.currentjobs = set() self.obtainSystemConstants() self.jobIDs = dict() self.lsfJobIDs = dict() self.nextJobID = 0 self.newJobsQueue = Queue() self.updatedJobsQueue = Queue() self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self) self.worker.setDaemon(True) self.worker.start() def __des__(self): #Closes the file handle associated with the results file. self.lsfResultsFileHandle.close( ) #Close the results file, cos were done. def issueBatchJob(self, jobNode): jobID = self.nextJobID self.nextJobID += 1 self.currentjobs.add(jobID) bsubline = prepareBsub(jobNode.cores, jobNode.memory, jobNode.jobName) + [jobNode.command] self.newJobsQueue.put((jobID, bsubline)) time.sleep(5) logger.debug("Issued the job command: %s with job id: %s " % (jobNode.command, str(jobID))) return jobID def getLsfID(self, jobID): if not jobID in self.lsfJobIDs: RuntimeError("Unknown jobID, could not be converted") (job, task) = self.lsfJobIDs[jobID] if task is None: return str(job) else: return str(job) + "." + str(task) def killBatchJobs(self, jobIDs): """Kills the given job IDs. """ for jobID in jobIDs: logger.debug("DEL: " + str(self.getLsfID(jobID))) self.currentjobs.remove(jobID) process = subprocess.Popen(["bkill", self.getLsfID(jobID)]) del self.jobIDs[self.lsfJobIDs[jobID]] del self.lsfJobIDs[jobID] toKill = set(jobIDs) while len(toKill) > 0: for jobID in list(toKill): if getjobexitcode(self.lsfJobIDs[jobID]) is not None: toKill.remove(jobID) if len(toKill) > 0: logger.warn( "Tried to kill some jobs, but something happened and they are still going, " "so I'll try again") time.sleep(5) def getIssuedBatchJobIDs(self): """A list of jobs (as jobIDs) currently issued (may be running, or maybe just waiting). """ return self.currentjobs def getRunningBatchJobIDs(self): """Gets a map of jobs (as jobIDs) currently running (not just waiting) and a how long they have been running for (in seconds). """ times = {} currentjobs = set() for x in self.getIssuedBatchJobIDs(): if x in self.lsfJobIDs: currentjobs.add(self.lsfJobIDs[x]) else: #not yet started pass process = subprocess.Popen(["bjobs"], stdout=subprocess.PIPE) for curline in process.stdout: items = curline.strip().split() if (len(items) > 9 and (items[0]) in currentjobs) and items[2] == 'RUN': jobstart = "/".join(items[7:9]) + '/' + str(date.today().year) jobstart = jobstart + ' ' + items[9] jobstart = time.mktime( time.strptime(jobstart, "%b/%d/%Y %H:%M")) jobstart = time.mktime( time.strptime(jobstart, "%m/%d/%Y %H:%M:%S")) times[self.jobIDs[(items[0])]] = time.time() - jobstart return times def getUpdatedBatchJob(self, maxWait): try: sgeJobID, retcode = self.updatedJobsQueue.get(timeout=maxWait) self.updatedJobsQueue.task_done() jobID, retcode = (self.jobIDs[sgeJobID], retcode) self.currentjobs -= {self.jobIDs[sgeJobID]} except Empty: pass else: return jobID, retcode, None def getWaitDuration(self): """We give parasol a second to catch its breath (in seconds) """ #return 0.0 return 15 @classmethod def getRescueBatchJobFrequency(cls): """Parasol leaks jobs, but rescuing jobs involves calls to parasol list jobs and pstat2, making it expensive. We allow this every 10 minutes.. """ return 1800 def shutdown(self): pass def obtainSystemConstants(self): p = subprocess.Popen(["lshosts"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) line = p.stdout.readline() items = line.strip().split() num_columns = len(items) cpu_index = None mem_index = None for i in range(num_columns): if items[i] == 'ncpus': cpu_index = i elif items[i] == 'maxmem': mem_index = i if cpu_index is None or mem_index is None: RuntimeError( "lshosts command does not return ncpus or maxmem columns") p.stdout.readline() self.maxCPU = 0 self.maxMEM = MemoryString("0") for line in p.stdout: items = line.strip().split() if len(items) < num_columns: RuntimeError("lshosts output has a varying number of columns") if items[cpu_index] != '-' and items[cpu_index] > self.maxCPU: self.maxCPU = items[cpu_index] if items[mem_index] != '-' and MemoryString( items[mem_index]) > self.maxMEM: self.maxMEM = MemoryString(items[mem_index]) if self.maxCPU is 0 or self.maxMEM is 0: RuntimeError("lshosts returns null ncpus or maxmem info") logger.debug("Got the maxCPU: %s" % (self.maxMEM))
class MyApp(object): instances = 0 def __init__(self, filename=None): MyApp.instances += 1 self.file_dlg = None self.info = None self.page_id = 0 self.has_hadith_numbers = False self.has_ayat = False self.keep_running = True self.queue = Queue() thread = Thread(target=self.worker_loop) thread.daemon = True thread.start() builder = Gtk.Builder() builder.add_from_file(GLADE_FN) builder.connect_signals(self) self.window = builder.get_object("main_win") self.header = builder.get_object("header") self.body = builder.get_object("body") self.toc_store = builder.get_object("toc_store") self.toc_tree = builder.get_object("toc_tree") self.search_entry = builder.get_object("search_entry") self.popover1 = builder.get_object("popover1") self.search_btn = builder.get_object("search_btn") self.page_btn = builder.get_object("page_btn") self.hadith_btn = builder.get_object("hadith_btn") self.aya_btn = builder.get_object("aya_btn") self.popover1_box = builder.get_object("popover1_box") self.search_menu_btns = [ self.search_btn, self.page_btn, self.hadith_btn, self.aya_btn ] self.window.show() if filename is not None: self.open(filename) def on_search_entry_key_release_event(self, w, event): __, keyval = event.get_keyval() if keyval == Gdk.KEY_Down or keyval == Gdk.KEY_KP_Down: self.update_search() ch = self.popover1_box.get_children() if ch: ch[0].grab_focus() elif keyval == Gdk.KEY_Escape: self.popover1.popdown() def on_search_entry_focus_out_event(self, w, event): if not self.popover1.get_focus_child(): self.popover1.popdown() def update_search(self): txt = touni(self.search_entry.get_text()).strip() if not txt: self.popover1.popdown() return entered_i = try_int(txt) m = sura_aya_re.match(txt) if self.has_ayat else None if m: for btn in self.search_menu_btns: btn.hide() self.aya_btn.show() self.aya_btn.set_label(u"سورة {} آية {}".format( m.group(1), m.group(2))) elif entered_i is None: for btn in self.search_menu_btns: btn.hide() self.search_btn.show() self.search_btn.set_label(u"البحث عن [{}]".format(txt)) self.popover1_box.reorder_child(self.search_btn, 0) else: self.search_btn.show() self.page_btn.show() self.hadith_btn.show() if self.has_ayat: self.aya_btn.show() else: self.aya_btn.hide() self.page_btn.set_label(u"صفحة [{}]".format(txt)) self.hadith_btn.set_label(u"حديث رقم [{}]".format(txt)) self.aya_btn.set_label(u"سورة رقم [{}]".format(txt)) if self.has_hadith_numbers: self.popover1_box.reorder_child(self.hadith_btn, 0) self.popover1_box.reorder_child(self.page_btn, 1) else: self.popover1_box.reorder_child(self.page_btn, 0) self.popover1_box.reorder_child(self.hadith_btn, 1) self.search_btn.set_label(u"البحث عن [{}]".format(txt)) self.popover1.set_relative_to(self.search_entry) self.popover1.popup() def on_search_entry_changed(self, w): self.update_search() def on_info_btn_clicked(self, w): self.goto_page(0) def on_previous_btn_clicked(self, w): self.goto_page(max(0, self.page_id - 1)) def on_next_btn_clicked(self, w): self.goto_page(self.page_id + 1) def on_search_entry_activate(self, w): self.popover1.popdown() text = w.get_text() page_id = try_int(text) if page_id is not None: self.goto_page(page_id) def worker_loop(self): while self.keep_running: try: a = self.queue.get(timeout=10) except Empty: continue cb_name, kwargs = a cb = getattr(self, cb_name) if not cb: self.queue.task_done() continue try: cb(**kwargs) except Exception as e: logger.error("ERROR: %r", e) self.queue.task_done() logger.info("worker thread exited") def open(self, filename): self.filename = filename cols = get_table_col(filename, 'Main') self.db = db = pyodbc.connect( tob('DRIVER=libmdbodbc.so;DBQ={}'.format(filename)), readonly=True, ansi=True, unicode_results=False, ) cursor = db.cursor() cursor.execute(u'SELECT {} FROM Main'.format(','.join(cols))) self.info = row_to_dict(cursor.fetchone(), cols) self.header.set_title(self.info['Bk']) self.goto_page(0) #cols = cursor.columns('Main') # does not work self.id = int(self.info['BkId']) cursor = db.cursor() tbl_body = 'b{}'.format(self.info['BkId']) cols = get_table_col(self.filename, tbl_body) if 'Hno' in cols: # take sample of 100 page to see if it has Hno cursor.execute(u'SELECT Hno FROM {}'.format(tbl_body)) for i in range(100): r = cursor.fetchone() if r is None: break if try_int(r[0]) is not None: print("has Hno") self.has_hadith_numbers = True break if 'sora' in cols and 'aya' in cols: # take sample of 100 page to see if it has ayat cursor.execute(u'SELECT sora, aya FROM {}'.format(tbl_body)) for i in range(100): r = cursor.fetchone() if r is None: break if try_int(r[0]) is not None and try_int(r[1]) is not None: print("has ayat") self.has_ayat = True break tbl_toc = 't{}'.format(self.info['BkId']) cols = get_table_col(filename, tbl_toc) cursor.execute(u'SELECT {} FROM {}'.format(','.join(cols), tbl_toc)) rows = [row_to_dict(row, cols) for row in cursor.fetchall()] rows.sort(key=lambda r: (r['id'], r['sub'])) def cb(r): t1 = time.time() parents_stack = [None] levels_stack = [] loop_it = iter(r) while (True): l = list(islice(loop_it, 1000)) if not l: break for row in r: level = row['lvl'] while (levels_stack and levels_stack[-1] >= level): levels_stack.pop() parents_stack.pop() it = self.toc_store.append(parents_stack[-1], ( row['tit'], row['lvl'], row['sub'], row['id'], )) parents_stack.append(it) levels_stack.append(level) Gtk.main_iteration_do(False) logger.info('building toc took %r', time.time() - t1) # it's a store, not UI, so we might be able to edit it directly # cb(rows) # if not then it's added like this GLib.idle_add(cb, rows) def goto_page(self, page_id, move_toc=False): if self.info is None: return self.page_id = page_id if page_id == 0: text = self.info['Betaka'] else: tbl_body = 'b{}'.format(self.info['BkId']) cols = get_table_col(self.filename, tbl_body) cursor = self.db.cursor() cursor.execute(u'SELECT {} FROM {} WHERE id={}'.format( ','.join(cols), tbl_body, page_id)) self.page = row_to_dict(cursor.fetchone(), cols) text = self.page['nass'] GLib.idle_add(lambda: self.body.get_buffer().set_text(text)) def on_window_destroy(self, w): self.keep_running = False MyApp.instances -= 1 logger.info("running instances = %r", MyApp.instances) if MyApp.instances == 0: Gtk.main_quit() def on_toc_tree_selection_changed(self, w): s, i = w.get_selected() # can be accessed in many ways row=tuple(s[i]) or id=s[i][3] or id=s.get_value(i, 3) self.queue.put(( 'goto_page', { 'page_id': s[i][3] }, )) def on_open_btn_clicked(self, w): filename = get_filename(self.window) if filename: if self.info is None: self.queue.put(( 'open', { 'filename': filename }, )) else: spawn_clone(filename)
class GraphiteReporter(threading.Thread): """A graphite reporter thread.""" def __init__(self, host, port, maxQueueSize=10000): """Connect to a Graphite server on host:port.""" threading.Thread.__init__(self) self.host, self.port = host, port self.sock = None self.queue = Queue() self.maxQueueSize = maxQueueSize self.daemon = True def run(self): """Run the thread.""" while True: try: try: name, value, valueType, stamp = self.queue.get() except TypeError: break self.log(name, value, valueType, stamp) finally: self.queue.task_done() def connect(self): """Connects to the Graphite server if not already connected.""" if self.sock is not None: return backoff = 0.01 while True: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(5) sock.connect((self.host, self.port)) self.sock = sock return except socket.error: time.sleep(random.uniform(0, 2.0*backoff)) backoff = min(backoff*2.0, 5.0) def disconnect(self): """Disconnect from the Graphite server if connected.""" if self.sock is not None: try: self.sock.close() except socket.error: pass finally: self.sock = None def _sendMsg(self, msg): """Send a line to graphite. Retry with exponential backoff.""" if not self.sock: self.connect() if not isinstance(msg, binary_type): msg = msg.encode("UTF-8") backoff = 0.001 while True: try: self.sock.sendall(msg) break except socket.error: log.warning('Graphite connection error', exc_info = True) self.disconnect() time.sleep(random.uniform(0, 2.0*backoff)) backoff = min(backoff*2.0, 5.0) self.connect() def _sanitizeName(self, name): """Sanitize a metric name.""" return name.replace(' ', '-') def log(self, name, value, valueType=None, stamp=None): """Log a named numeric value. The value type may be 'value', 'count', or None.""" if type(value) == float: form = "%s%s %2.2f %d\n" else: form = "%s%s %s %d\n" if valueType is not None and len(valueType) > 0 and valueType[0] != '.': valueType = '.' + valueType if not stamp: stamp = time.time() self._sendMsg(form % (self._sanitizeName(name), valueType or '', value, stamp)) def enqueue(self, name, value, valueType=None, stamp=None): """Enqueue a call to log.""" # If queue is too large, refuse to log. if self.maxQueueSize and self.queue.qsize() > self.maxQueueSize: return # Stick arguments into the queue self.queue.put((name, value, valueType, stamp)) def flush(self): """Block until all stats have been sent to Graphite.""" self.queue.join() def shutdown(self): """Shut down the background thread.""" self.queue.put(None) self.flush()
class TCPClientManager(object): """A Client for the 'Push' feature in Device Cloud""" def __init__(self, conn, secure=True, ca_certs=None, workers=1): """ Arbitrator for multiple TCP Client Sessions :param conn: The :class:`devicecloud.DeviceCloudConnection` to use :param secure: Whether or not to create a secure SSL wrapped session. :param ca_certs: Path to a file containing Certificates. If not provided, the devicecloud.crt file provided with the module will be used. In most cases, the devicecloud.crt file should be acceptable. :param workers: Number of workers threads to process callback calls. """ self._conn = conn self._secure = secure self._ca_certs = ca_certs # A dict mapping Sockets to their PushSessions self.sessions = {} # IO thread is used monitor sockets and consume data. self._io_thread = None # Writer thread is used to send data on sockets. self._writer_thread = None # Write queue is used to queue up data to write to sockets. self._write_queue = Queue() # A pool that monitors callback events and invokes them. self._callback_pool = CallbackWorkerPool(self._write_queue, size=workers) self.closed = False self.log = logging.getLogger(__name__) @property def hostname(self): return self._conn.hostname @property def username(self): return self._conn.username @property def password(self): return self._conn.password def _restart_session(self, session): """Restarts and re-establishes session :param session: The session to restart """ # remove old session key, if socket is None, that means the # session was closed by user and there is no need to restart. if session.socket is not None: self.log.info("Attempting restart session for Monitor Id %s." % session.monitor_id) del self.sessions[session.socket.fileno()] session.stop() session.start() self.sessions[session.socket.fileno()] = session def _writer(self): """ Indefinitely checks the writer queue for data to write to socket. """ while not self.closed: try: sock, data = self._write_queue.get(timeout=0.1) self._write_queue.task_done() sock.send(data) except Empty: pass # nothing to write after timeout except socket.error as err: if err.errno == errno.EBADF: self._clean_dead_sessions() def _clean_dead_sessions(self): """ Traverses sessions to determine if any sockets were removed (indicates a stopped session). In these cases, remove the session. """ for sck in list(self.sessions.keys()): session = self.sessions[sck] if session.socket is None: del self.sessions[sck] def _select(self): """ While the client is not marked as closed, performs a socket select on all PushSession sockets. If any data is received, parses and forwards it on to the callback function. If the callback is successful, a PublishMessageReceived message is sent. """ try: while not self.closed: try: inputready = select.select(self.sessions.keys(), [], [], 0.1)[0] for sock in inputready: session = self.sessions[sock] sck = session.socket if sck is None: # Socket has since been deleted, continue continue # If no defined message length, nothing has been # consumed yet, parse the header. if session.message_length == 0: # Read header information before receiving rest of # message. response_type = _read_msg_header(session) if response_type == NO_DATA: # No data could be read, assume socket closed. if session.socket is not None: self.log.error("Socket closed for Monitor %s." % session.monitor_id) self._restart_session(session) continue elif response_type == INCOMPLETE: # More Data to be read. Continue. continue elif response_type != PUBLISH_MESSAGE: self.log.warn("Response Type (%x) does not match PublishMessage (%x)" % (response_type, PUBLISH_MESSAGE)) continue try: if not _read_msg(session): # Data not completely read, continue. continue except PushException as err: # If Socket is None, it was closed, # otherwise it was closed when it shouldn't # have been restart it. session.data = six.b("") session.message_length = 0 if session.socket is None: del self.sessions[sck] else: self.log.exception(err) self._restart_session(session) continue # We received full payload, # clear session data and parse it. data = session.data session.data = six.b("") session.message_length = 0 block_id = struct.unpack('!H', data[0:2])[0] compression = struct.unpack('!B', data[4:5])[0] payload = data[10:] if compression == 0x01: # Data is compressed, uncompress it. payload = zlib.decompress(payload) # Enqueue payload into a callback queue to be # invoked self._callback_pool.queue_callback(session, block_id, payload) except select.error as err: # Evaluate sessions if we get a bad file descriptor, if # socket is gone, delete the session. if err.args[0] == errno.EBADF: self._clean_dead_sessions() except Exception as err: self.log.exception(err) finally: for session in self.sessions.values(): if session is not None: session.stop() def _init_threads(self): """Initializes the IO and Writer threads""" if self._io_thread is None: self._io_thread = Thread(target=self._select) self._io_thread.start() if self._writer_thread is None: self._writer_thread = Thread(target=self._writer) self._writer_thread.start() def create_session(self, callback, monitor_id): """ Creates and Returns a PushSession instance based on the input monitor and callback. When data is received, callback will be invoked. If neither monitor or monitor_id are specified, throws an Exception. :param callback: Callback function to call when PublishMessage messages are received. Expects 1 argument which will contain the payload of the pushed message. Additionally, expects function to return True if callback was able to process the message, False or None otherwise. :param monitor_id: The id of the Monitor, will be queried to understand parameters of the monitor. """ self.log.info("Creating Session for Monitor %s." % monitor_id) session = SecurePushSession(callback, monitor_id, self, self._ca_certs) \ if self._secure else PushSession(callback, monitor_id, self) session.start() self.sessions[session.socket.fileno()] = session self._init_threads() return session def stop(self): """Stops all session activity. Blocks until io and writer thread dies """ if self._io_thread is not None: self.log.info("Waiting for I/O thread to stop...") self.closed = True self._io_thread.join() if self._writer_thread is not None: self.log.info("Waiting for Writer Thread to stop...") self.closed = True self._writer_thread.join() self.log.info("All worker threads stopped.")
class CallbackWorkerPool(object): """ A Worker Pool implementation that creates a number of predefined threads used for invoking Session callbacks. """ def __init__(self, write_queue=None, size=1): """ Creates a Callback Worker Pool for use in invoking Session Callbacks when data is received by a push client. :param write_queue: Queue used for queueing up socket write events for when a payload message is received and processed. :param size: The number of worker threads to invoke callbacks. """ # Used to queue up PublishMessageReceived events to be sent back to # the iDigi server. self._write_queue = write_queue # Used to queue up sessions and data to callback with. self._queue = Queue(size) # Number of workers to create. self.size = size self.log = logging.getLogger('{}.callback_worker_pool'.format(__name__)) for _ in range(size): worker = Thread(target=self._consume_queue) worker.daemon = True worker.start() def _consume_queue(self): """ Continually blocks until data is on the internal queue, then calls the session's registered callback and sends a PublishMessageReceived if callback returned True. """ while True: session, block_id, raw_data = self._queue.get() data = json.loads(raw_data.decode('utf-8')) # decode as JSON try: result = session.callback(data) if result is None: self.log.warn("Callback %r returned None, expected boolean. Messages " "are not marked as received unless True is returned", session.callback) elif result: # Send a Successful PublishMessageReceived with the # block id sent in request if self._write_queue is not None: response_message = struct.pack('!HHH', PUBLISH_MESSAGE_RECEIVED, block_id, 200) self._write_queue.put((session.socket, response_message)) except Exception as exception: self.log.exception(exception) self._queue.task_done() def queue_callback(self, session, block_id, data): """ Queues up a callback event to occur for a session with the given payload data. Will block if the queue is full. :param session: the session with a defined callback function to call. :param block_id: the block_id of the message received. :param data: the data payload of the message received. """ self._queue.put((session, block_id, data))
class LSFBatchSystem(BatchSystemSupport): """ The interface for running jobs on lsf, runs all the jobs you give it as they come in, but in parallel. """ @classmethod def supportsWorkerCleanup(cls): return False @classmethod def supportsHotDeployment(cls): return False def shutdown(self): pass def __init__(self, config, maxCores, maxMemory, maxDisk): super(LSFBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk) self.lsfResultsFile = self._getResultsFileName(config.jobStore) #Reset the job queue and results (initially, we do this again once we've killed the jobs) self.lsfResultsFileHandle = open(self.lsfResultsFile, 'w') self.lsfResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence self.currentjobs = set() self.obtainSystemConstants() self.jobIDs = dict() self.lsfJobIDs = dict() self.nextJobID = 0 self.newJobsQueue = Queue() self.updatedJobsQueue = Queue() self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self) self.worker.setDaemon(True) self.worker.start() def __des__(self): #Closes the file handle associated with the results file. self.lsfResultsFileHandle.close() #Close the results file, cos were done. def issueBatchJob(self, jobNode): jobID = self.nextJobID self.nextJobID += 1 self.currentjobs.add(jobID) bsubline = prepareBsub(jobNode.cores, jobNode.memory) + [jobNode.command] self.newJobsQueue.put((jobID, bsubline)) logger.debug("Issued the job command: %s with job id: %s " % (jobNode.command, str(jobID))) return jobID def getLsfID(self, jobID): if not jobID in self.lsfJobIDs: RuntimeError("Unknown jobID, could not be converted") (job,task) = self.lsfJobIDs[jobID] if task is None: return str(job) else: return str(job) + "." + str(task) def killBatchJobs(self, jobIDs): """Kills the given job IDs. """ for jobID in jobIDs: logger.debug("DEL: " + str(self.getLsfID(jobID))) self.currentjobs.remove(jobID) process = subprocess.Popen(["bkill", self.getLsfID(jobID)]) del self.jobIDs[self.lsfJobIDs[jobID]] del self.lsfJobIDs[jobID] toKill = set(jobIDs) while len(toKill) > 0: for jobID in list(toKill): if getjobexitcode(self.lsfJobIDs[jobID]) is not None: toKill.remove(jobID) if len(toKill) > 0: logger.warn("Tried to kill some jobs, but something happened and they are still going, " "so I'll try again") time.sleep(5) def getIssuedBatchJobIDs(self): """A list of jobs (as jobIDs) currently issued (may be running, or maybe just waiting). """ return self.currentjobs def getRunningBatchJobIDs(self): """Gets a map of jobs (as jobIDs) currently running (not just waiting) and a how long they have been running for (in seconds). """ times = {} currentjobs = set() for x in self.getIssuedBatchJobIDs(): if x in self.lsfJobIDs: currentjobs.add(self.lsfJobIDs[x]) else: #not yet started pass process = subprocess.Popen(["bjobs"], stdout = subprocess.PIPE) for curline in process.stdout: items = curline.strip().split() if (len(items) > 9 and (items[0]) in currentjobs) and items[2] == 'RUN': jobstart = "/".join(items[7:9]) + '/' + str(date.today().year) jobstart = jobstart + ' ' + items[9] jobstart = time.mktime(time.strptime(jobstart,"%b/%d/%Y %H:%M")) jobstart = time.mktime(time.strptime(jobstart,"%m/%d/%Y %H:%M:%S")) times[self.jobIDs[(items[0])]] = time.time() - jobstart return times def getUpdatedBatchJob(self, maxWait): try: sgeJobID, retcode = self.updatedJobsQueue.get(timeout=maxWait) self.updatedJobsQueue.task_done() jobID, retcode = (self.jobIDs[sgeJobID], retcode) self.currentjobs -= {self.jobIDs[sgeJobID]} except Empty: pass else: return jobID, retcode, None def getWaitDuration(self): """We give parasol a second to catch its breath (in seconds) """ #return 0.0 return 15 @classmethod def getRescueBatchJobFrequency(cls): """Parasol leaks jobs, but rescuing jobs involves calls to parasol list jobs and pstat2, making it expensive. We allow this every 10 minutes.. """ return 1800 def obtainSystemConstants(self): p = subprocess.Popen(["lshosts"], stdout = subprocess.PIPE, stderr = subprocess.STDOUT) line = p.stdout.readline() items = line.strip().split() num_columns = len(items) cpu_index = None mem_index = None for i in range(num_columns): if items[i] == 'ncpus': cpu_index = i elif items[i] == 'maxmem': mem_index = i if cpu_index is None or mem_index is None: RuntimeError("lshosts command does not return ncpus or maxmem columns") p.stdout.readline() self.maxCPU = 0 self.maxMEM = MemoryString("0") for line in p.stdout: items = line.strip().split() if len(items) < num_columns: RuntimeError("lshosts output has a varying number of columns") if items[cpu_index] != '-' and items[cpu_index] > self.maxCPU: self.maxCPU = items[cpu_index] if items[mem_index] != '-' and MemoryString(items[mem_index]) > self.maxMEM: self.maxMEM = MemoryString(items[mem_index]) if self.maxCPU is 0 or self.maxMEM is 0: RuntimeError("lshosts returns null ncpus or maxmem info") logger.debug("Got the maxCPU: %s" % (self.maxMEM))
class Rigger(object): """ A Rigger event framework instance. The Rigger object holds all configuration and instances of plugins. By default Rigger accepts a configuration file name to parse, though it is perfectly acceptable to pass the configuration into the ``self.config`` attribute. Args: config_file: A configuration file holding all of Riggers base and plugin configuration. """ def __init__(self, config_file): self.gdl = threading.Lock() self.pre_callbacks = defaultdict(dict) self.post_callbacks = defaultdict(dict) self.plugins = {} self.config_file = config_file self.squash_exceptions = False self.initialized = False self._task_list = {} self._queue_lock = threading.Lock() self._global_queue = Queue() self._background_queue = Queue() self._server_shutdown = False self._zmq_event_handler_shutdown = False self._global_queue_shutdown = False self._background_queue_shutdown = False globt = threading.Thread(target=self.process_queue, name="global_queue_processor") globt.start() bgt = threading.Thread( target=self.process_background_queue, name="background_queue_processor") bgt.start() def process_queue(self): """ The ``process_queue`` thread manages taking events on and off of the global queue. Both TCP and in-object fire_hooks place events onto the global_queue and these are both handled by the same handler called ``process_hook``. If there is an exception during processing, the exception is printed and execution continues. """ while not self._global_queue_shutdown: while not self._global_queue.empty(): with self._queue_lock: tid = self._global_queue.get() obj = self._task_list[tid].json_dict self._task_list[tid].status = Task.RUNNING try: loc, glo = self.process_hook(obj['hook_name'], **obj['data']) combined_dict = {} combined_dict.update(glo) combined_dict.update(loc) self._task_list[tid].output = combined_dict except Exception as e: self.log_message(e) with self._queue_lock: self._global_queue.task_done() self._task_list[tid].status = Task.FINISHED if not self._task_list[tid].json_dict.get('grab_result', None): del self._task_list[tid] time.sleep(0.1) def process_background_queue(self): """ The ``process_background_queue`` manages the hooks which have been backgrounded. In this respect the tasks that are completed are not required to continue with the test and as such can be forgotten about. An example of this would be some that sends an email, or tars up files, it has all the information it needs and the main process doesn't need to wait for it to complete. """ while not self._background_queue_shutdown: while not self._background_queue.empty(): obj = self._background_queue.get() try: local, globals_updates = self.process_callbacks(obj['cb'], obj['kwargs']) with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) except Exception as e: self.log_message(e) self._background_queue.task_done() time.sleep(0.1) def zmq_event_handler(self, zmq_socket_address): """ The ``zmq_event_handler`` thread receives (and responds to) updates from the zmq socket, which is normally embedded in the web server running alongside this riggerlib instance, in its own process. """ ctx = zmq.Context() zmq_socket = ctx.socket(zmq.REP) zmq_socket.set(zmq.RCVTIMEO, 300) zmq_socket.bind(zmq_socket_address) def zmq_reply(message, **extra): payload = {'message': message} payload.update(extra) zmq_socket.send_json(payload) bad_request = partial(zmq_reply, 'BAD REQUEST') while not self._zmq_event_handler_shutdown: try: json_dict = zmq_socket.recv_json() except zmq.Again: continue try: event_name = json_dict['event_name'] except KeyError: bad_request() if event_name == 'fire_hook': tid = self._fire_internal_hook(json_dict) if tid: zmq_reply('OK', tid=tid) else: bad_request() elif event_name == 'task_check': try: tid = json_dict['tid'] extra = { "tid": tid, "status": self._task_list[tid].status, } if json_dict['grab_result']: extra["output"] = self._task_list[tid].output zmq_reply('OK', **extra) except KeyError: zmq_reply('NOT FOUND') elif event_name == 'task_delete': try: tid = json_dict['tid'] del self._task_list[tid] zmq_reply('OK', tid=tid) except KeyError: zmq_reply('OK', tid=tid) elif event_name == 'shutdown': zmq_reply('OK') # We gotta initiate server stop from here and stop this thread self._server_shutdown = True break elif event_name == 'ping': zmq_reply('PONG') else: bad_request() zmq_socket.close() def read_config(self, config_file): """ Reads in the config file and parses the yaml data. Args: config_file: A configuration file holding all of Riggers base and plugin configuration. Raises: IOError: If the file can not be read. Exception: If there is any error parsing the configuration file. """ try: with open(config_file, "r") as stream: data = yaml.load(stream) except IOError: print("!!! Configuration file could not be loaded...exiting") sys.exit(127) except Exception as e: print(e) print("!!! Error parsing Configuration file") sys.exit(127) self.config = data def parse_config(self): """ Takes the configuration data from ``self.config`` and sets up the plugin instances. """ self.read_config(self.config_file) self.setup_plugin_instances() self.start_server() def setup_plugin_instances(self): """ Sets up instances into a dict called ``self.instances`` and instantiates each instance of the plugin. It also sets the ``self._threaded`` option to determine if plugins will be processed synchronously or asynchronously. """ self.instances = {} self._threaded = self.config.get("threaded", False) plugins = self.config.get("plugins", {}) for ident, config in plugins.items(): self.setup_instance(ident, config) def setup_instance(self, ident, config): """ Sets up a single instance into the ``self.instances`` dict. If the instance does not exist, a warning is printed out. Args: ident: A plugin instance identifier. config: Configuration dict from the yaml. """ plugin_name = config.get('plugin', {}) if plugin_name in self.plugins: obj = self.plugins[plugin_name] if obj: obj_instance = obj(ident, config, self) self.instances[ident] = RiggerPluginInstance(ident, obj_instance, config) else: msg = "Plugin [{}] was not found, "\ "disabling instance [{}]".format(plugin_name, ident) self.log_message(msg) def start_server(self): """ Starts the ZMQ server if the ``server_enabled`` is True in the config. """ self._server_hostname = self.config.get('server_address', '127.0.0.1') self._server_port = self.config.get('server_port', 21212) self._server_enable = self.config.get('server_enabled', False) if self._server_enable: zmq_socket_address = 'tcp://{}:{}'.format(self._server_hostname, self._server_port) # set up reciever thread for zmq event handling zeh = threading.Thread( target=self.zmq_event_handler, args=(zmq_socket_address,), name="zmq_event_handler") zeh.start() exect = threading.Thread(target=self.await_shutdown, name="executioner") exect.start() def await_shutdown(self): while not self._server_shutdown: time.sleep(0.3) self.stop_server() def stop_server(self): """ Responsible for the following: - stopping the zmq event handler (unless already stopped through 'terminate') - stopping the global queue - stopping the background queue """ self.log_message("Shutdown initiated : {}".format(self._server_hostname)) # The order here is important self._zmq_event_handler_shutdown = True self._global_queue.join() self._global_queue_shutdown = True self._background_queue.join() self._background_queue_shutdown = True raise SystemExit def fire_hook(self, hook_name, **kwargs): """ Parses the hook information into a dict for passing to process_hook. This is used to enable both the TCP and in-object fire_hook methods to use the same process_hook method call. Args: hook_name: The name of the hook to fire. kwargs: The kwargs to pass to the hooks. """ json_dict = {'hook_name': hook_name, 'data': kwargs} self._fire_internal_hook(json_dict) def _fire_internal_hook(self, json_dict): task = Task(json_dict) tid = task.tid.hexdigest() self._task_list[tid] = task if self._global_queue: with self._queue_lock: self._global_queue.put(tid) return tid else: return None def process_hook(self, hook_name, **kwargs): """ Takes a hook_name and a selection of kwargs and fires off the appropriate callbacks. This function is the guts of Rigger and is responsible for running the callback and hook functions. It first loads some blank dicts to collect the updates for the local and global namespaces. After this, it loads the pre_callback functions along with the kwargs into the callback collector processor. The return values are then classifed into local and global dicts and updates proceed. After this, the plugin hooks themselves are then run using the same methodology. Their return values are merged with the existing dicts and then the same process happens for the post_callbacks. Note: If the instance of the plugin has been marked as a background instance, and hooks which are called in that instance will be backgrounded. The hook will also not be able to return any data to the post-hook callback, although updates to globals will be processed as and when the backgrounded task is completed. Args: hook_name: The name of the hook to fire. kwargs: The kwargs to pass to the hooks. """ if not self.initialized: return kwargs_updates = {} globals_updates = {} kwargs.update({'config': self.config}) # First fire off any pre-hook callbacks if self.pre_callbacks.get(hook_name): # print "Running pre hook callback for {}".format(hook_name) kwargs_updates, globals_updates = self.process_callbacks( self.pre_callbacks[hook_name].values(), kwargs) # Now we can update the kwargs passed to the real hook with the updates with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) kwargs = recursive_update(kwargs, kwargs_updates) # Now fire off each plugin hook event_hooks = [] for instance_name, instance in self.instances.items(): callbacks = instance.obj.callbacks enabled = instance.data.get('enabled', None) if callbacks.get(hook_name) and enabled: cb = callbacks[hook_name] if instance.data.get('background', False): self._background_queue.put({'cb': [cb], 'kwargs': kwargs}) elif cb['bg']: self._background_queue.put({'cb': [cb], 'kwargs': kwargs}) else: event_hooks.append(cb) kwargs_updates, globals_updates = self.process_callbacks(event_hooks, kwargs) # One more update for the post_hook callback with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) kwargs = recursive_update(kwargs, kwargs_updates) # Finally any post-hook callbacks if self.post_callbacks.get(hook_name): # print "Running post hook callback for {}".format(hook_name) kwargs_updates, globals_updates = self.process_callbacks( self.post_callbacks[hook_name].values(), kwargs) with self.gdl: self.global_data = recursive_update(self.global_data, globals_updates) kwargs = recursive_update(kwargs, kwargs_updates) return kwargs, self.global_data def process_callbacks(self, callback_collection, kwargs): """ Processes a collection of callbacks or hooks for a particular event, namely pre, hook or post. The functions are passed in as an array to ``callback_collection`` and process callbacks first iterates each function and ensures that each one has the correct arguments available to it. If not, an Exception is raised. Then, depending on whether Threading is enabled or not, the functions are either run sequentially, or loaded into a ThreadPool and executed asynchronously. The returned local and global updates are either collected and processed sequentially, as in the case of the non-threaded behaviour, or collected at the end of the callback_collection processing and handled there. Note: It is impossible to predict the order of the functions being run. If the order is important, it is advised to create a second event hook that will be fired before the other. Rigger has no concept of hook or callback order and is unlikely to ever have. Args: callback_collection: A list of functions to call. kwargs: A set of kwargs to pass to the functions. Returns: A tuple of local and global namespace updates. """ loc_collect = {} glo_collect = {} if self._threaded: results_list = [] pool = ThreadPool(10) for cb in callback_collection: required_args = [sig for sig in cb['args'] if isinstance(cb['args'][sig].default, type)] missing = list(set(required_args).difference(set(self.global_data.keys())) .difference(set(kwargs.keys()))) if not missing: new_kwargs = self.build_kwargs(cb['args'], kwargs) if self._threaded: results_list.append(pool.apply_async(cb['func'], [], new_kwargs)) else: obtain_result = self.handle_results(cb['func'], [], new_kwargs) loc_collect, glo_collect = self.handle_collects( obtain_result, loc_collect, glo_collect) else: raise Exception('Function {} is missing kwargs {}' .format(cb['func'].__name__, missing)) if self._threaded: pool.close() pool.join() for result in results_list: obtain_result = self.handle_results(result.get, [], {}) loc_collect, glo_collect = self.handle_collects( obtain_result, loc_collect, glo_collect) return loc_collect, glo_collect def handle_results(self, call, args, kwargs): """ Handles results and depending on configuration, squashes exceptions and logs or returns the obtained result. Args: call: The function call. args: The positional arguments. kwargs: The keyword arguments. Returns: The obtained result of the callback or hook. """ try: obtain_result = call(*args, **kwargs) except: if self.squash_exceptions: obtain_result = None self.handle_failure(sys.exc_info()) else: raise return obtain_result def handle_collects(self, result, loc_collect, glo_collect): """ Handles extracting the information from the hook/callback result. If the hook/callback returns None, then the dicts are returned unaltered, else they are updated with local, global namespace updates. Args: result: The result to process. loc_collect: The local namespace updates collection. glo_collect: The global namespace updates collection. Returns: A tuple containing the local and global updates. """ if result: if result[0]: loc_collect = recursive_update(loc_collect, result[0]) if result[1]: glo_collect = recursive_update(glo_collect, result[1]) return loc_collect, glo_collect def build_kwargs(self, args, kwargs): """ Builds a new kwargs from a list of allowed args. Functions only receive a single set of kwargs, and so the global and local namespaces have to be collapsed. In this way, the local overrides the global namespace, hence if a key exists in both local and global, the local value will be passed to the function under the the key name and the global value will be forgotten. The args parameter ensures that only the expected arguments are supplied. Args: args: A list of allowed argument names kwargs: A dict of kwargs from the local namespace. Returns: A consolidated global/local namespace with local overrides. """ returned_args = {} returned_args.update({ name: self.global_data[name] for name in args if name in self.global_data}) returned_args.update({ name: kwargs[name] for name in args if name in kwargs}) return returned_args def register_hook_callback(self, hook_name=None, ctype="pre", callback=None, name=None): """ Registers pre and post callbacks. Takes a callback function and assigns it to the hook_name with an optional identifier. The optional identifier makes it possible to hot bind functions into hooks and to remove them at a later date with ``unregister_hook_callback``. Args: hook_name: The name of the event hook to respond to. ctype: The call back type, either ``pre`` or ``post``. callback: The callback function. name: An optional name for the callback instance binding. """ if hook_name and callback: callback_instance = self.create_callback(callback) if not name: name = hashlib.sha1( str(time.time()) + hook_name + str(callback_instance['args'])).hexdigest() if ctype == "pre": self.pre_callbacks[hook_name][name] = callback_instance elif ctype == "post": self.post_callbacks[hook_name][name] = callback_instance def unregister_hook_callback(self, hook_name, ctype, name): """ Unregisters a pre or post callback. If the binding has a known name, this function allows the removal of a binding. Args: hook_name: The event hook name. ctype: The callback type, either ``pre`` or ``post``. name: An optional name for the callback instance binding. """ if ctype == "pre": del self.pre_callbacks[hook_name][name] elif ctype == "post": del self.post_callbacks[hook_name][name] def register_plugin(self, cls, plugin_name=None): """ Registers a plugin class to a name. Multiple instances of the same plugin can be used in Rigger, ``self.plugins`` stores un-initialized class defintions to be used by ``setup_instances``. Args: cls: The class. plugin_name: The name of the plugin. """ if plugin_name in self.plugins: print("Plugin name already taken [{}]".format(plugin_name)) elif plugin_name is None: print("Plugin name cannot be None") else: # print "Registering plugin {}".format(plugin_name) self.plugins[plugin_name] = cls def get_instance_obj(self, name): """ Gets the instance object for a given ident name. Args: name: The ident name of the instance. Returns: The object of the instance. """ if name in self.instances: return self.instances[name].obj else: return None def get_instance_data(self, name): """ Gets the instance data(config) for a given ident name. Args: name: The ident name of the instance. Returns: The data(config) of the instance. """ if name in self.instances: return self.instances[name].data else: return None def configure_plugin(self, name, *args, **kwargs): """ Attempts to configure an instance, passing it the args and kwargs. Args: name: The ident name of the instance. args: The positional args. kwargs: The keyword arguments. """ obj = self.get_instance_obj(name) if obj: obj.configure(*args, **kwargs) @staticmethod def create_callback(callback, bg=False): """ Simple function to inspect a function and return it along with it param names wrapped up in a nice dict. This forms a callback object. Args: callback: The callback function. Returns: A dict of function and param names. """ params = signature(callback).parameters return { 'func': callback, 'args': params, 'bg': bg } def handle_failure(self, exc): """ Handles an exception. It is expected that this be overidden. """ self.log_message(exc) def log_message(self, message): """ "Logs" a message. It is expected that this be overidden. """ print(message)
class QueueWorker(BaseWorker): TIMEOUT_ATTEMPTS = 5 QUEUE_SIZE = -1 # inf END_EVENT = object() NAME = 'polyaxon.QueueWorker' def __init__(self, timeout=None, queue_size=None): super(QueueWorker, self).__init__() self._queue = Queue(queue_size or self.QUEUE_SIZE) self._timeout = timeout if timeout is not None else settings.TIMEOUT def atexit(self): with self._lock: if not self.is_alive(): return self._queue.put_nowait(self.END_EVENT) def timeout_join(timeout, queue): end = time() + timeout queue.all_tasks_done.acquire() try: while queue.unfinished_tasks: current_timeout = end - time() if current_timeout <= 0: # timed out return False queue.all_tasks_done.wait(timeout=current_timeout) return True finally: queue.all_tasks_done.release() # ensure wait timeout = min(settings.MIN_TIMEOUT, self._timeout / self.TIMEOUT_ATTEMPTS) if timeout_join(timeout=timeout, queue=self._queue): timeout = 0 else: # Queue still has message, try another time size = self._queue.qsize() if not settings.IN_CLUSTER: print( 'Polyaxon %s is attempting to send %i pending messages' % (self.NAME, size)) print('Waiting up to {} seconds'.format(self._timeout)) if os.name == 'nt': print('Press Ctrl-Break to quit') else: print('Press Ctrl-C to quit') sleep(settings.MIN_TIMEOUT) # Allow tasks to get executed while timeout > 0 and not timeout_join(timeout=timeout, queue=self._queue): timeout = min(timeout + self._timeout / self.TIMEOUT_ATTEMPTS, self._timeout - timeout) size = self._queue.qsize() if size > 0: print( 'Polyaxon %s timed out and did not manage to send %i messages' % (self.NAME, size)) self._thread = None def stop(self, timeout=None): with self._lock: if self._thread: self._queue.put_nowait(self.END_EVENT) self._thread.join(timeout=timeout) self._thread = None self._thread_for_pid = None def queue(self, callback, *args, **kwargs): self.is_running() self._queue.put_nowait((callback, args, kwargs)) def _target(self): while True: record = self._queue.get() try: if record is self.END_EVENT: break callback, args, kwargs = record try: callback(*args, **kwargs) except Exception: logger.error('Failed processing job', exc_info=True) finally: self._queue.task_done() sleep(0)
class QueuedRunner(object): """\ Secondary backend runner that puts method calls on a bounded queue and drops them when the queue is full. A separate (non-main) thread works the queue. """ def __init__(self): self.q = Queue(maxsize=100) self.worker_running = False def start_worker(self): def worker(): while True: (func, args, kwargs) = self.q.get() try: func(*args, **kwargs) metrics.incr( 'tagstore.multi.runner.execute', instance='success', skip_internal=True, ) except Exception as e: logger.exception(e) metrics.incr( 'tagstore.multi.runner.execute', instance='fail', skip_internal=True, ) finally: self.q.task_done() t = Thread(target=worker) t.setDaemon(True) t.start() self.worker_running = True def run(self, f, *args, **kwargs): if random.random() <= options.get('tagstore.multi-sampling'): if not self.worker_running: self.start_worker() try: self.q.put((f, args, kwargs), block=False) metrics.incr( 'tagstore.multi.runner.schedule', instance='put', skip_internal=True, ) except Full: metrics.incr( 'tagstore.multi.runner.schedule', instance='full', skip_internal=True, ) return else: metrics.incr( 'tagstore.multi.runner.schedule', instance='sampled', skip_internal=True, )
class ThreadedFifoBuffer(FifoBuffer): """ FIFO-in-memory connection inside dedicated thread. This is external-IO usable for Moler since it has it's own runner (thread) that can work in background and pull data from FIFO-mem connection. Usable for integration tests. """ def __init__(self, moler_connection, echo=True, name=None, logger_name=""): """Initialization of FIFO-mem-threaded connection.""" super(ThreadedFifoBuffer, self).__init__(moler_connection=moler_connection, echo=echo, name=name, logger_name=logger_name) self.pulling_thread = None self.injections = Queue() def open(self): """Start thread pulling data from FIFO buffer.""" ret = super(ThreadedFifoBuffer, self).open() done = threading.Event() self.pulling_thread = TillDoneThread(target=self.pull_data, done_event=done, kwargs={'pulling_done': done}) self.pulling_thread.start() self._log(msg="open {}".format(self), level=logging.INFO) self._notify_on_connect() return ret def close(self): """Stop pulling thread.""" if self.pulling_thread: self.pulling_thread.join() self.pulling_thread = None super(ThreadedFifoBuffer, self).close() self._log(msg="closed {}".format(self), level=logging.INFO) self._notify_on_disconnect() def inject(self, input_bytes, delay=0.0): """ Add bytes to end of buffer :param input_bytes: iterable of bytes to inject :param delay: delay before each inject :return: None """ for data in input_bytes: self.injections.put((data, delay)) if not delay: time.sleep(0.05) # give subsequent read() a chance to get data def _inject_deferred(self): if self.deferred_injections: for data, delay in self.deferred_injections: self.injections.put((data, delay)) self.deferred_injections = [] time.sleep(0.05) # give subsequent read() a chance to get data def pull_data(self, pulling_done): """Pull data from FIFO buffer.""" while not pulling_done.is_set(): self.read() # internally forwards to embedded Moler connection try: data, delay = self.injections.get_nowait() if delay: time.sleep(delay) self._inject(data) self.injections.task_done() except Empty: time.sleep(0.01) # give FIFO chance to get data
class TCPHandler(logging.Handler): ''' Python logging handler for sending JSON formatted messages over TCP, optionally wrapping the connection with TLSv1 ''' def __init__(self, host, port, ssl_ca_file=None): ''' Instantiate a TCPHandler with the intent of connecting to the given host (string) and port (int) with or without using SSL/TLSv1 ''' logging.Handler.__init__(self) self.host = host self.port = port self.ssl_ca_file = ssl_ca_file self.sock = None self.queue = Queue(LOG_QUEUE_SIZE) self.connect_wait = BACKOFF_INITIAL self.raiseExceptions = 0 self.hostname = socket.gethostname() if self.hostname.find('.') != -1: self.hostname = self.hostname.split('.', 1)[0] self.sender = threading.Thread(target=self.run) self.sender.setDaemon(True) self.sender.start() def connect(self): ''' Create a connection with the server, sleeping for some period of time if connection errors have occurred recently. ''' self.sock = socket.socket() if self.ssl_ca_file: self.sock = ssl.wrap_socket(self.sock, ssl_version=ssl.PROTOCOL_TLSv1, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ssl_ca_file) INTERNAL_LOG.debug('Connecting (backoff: %.03f)' % self.connect_wait) time.sleep(self.connect_wait) self.sock.connect((self.host, self.port)) def jsonify(self, record): ''' Translate a LogRecord instance into a json_event ''' timestamp = datetime.utcfromtimestamp(record.created) timestamp = timestamp.isoformat() fields = { 'level': record.levelname, 'filename': record.pathname, 'lineno': record.lineno, 'method': record.funcName, } if record.exc_info: fields['exception'] = str(record.exc_info) fields['traceback'] = format_exc(record.exc_info) log = { '@source_host': self.hostname, '@timestamp': timestamp, '@tags': [record.name], '@message': record.getMessage(), '@fields': fields, } return json.dumps(log) def emit(self, record): ''' Send a LogRecord object formatted as json_event via a queue and worker thread. ''' self.queue.put_nowait(record) def run(self): ''' Main loop of the logger thread. All network I/O and exception handling originates here. Strings are consumed from self.queue and sent to self.sock, creating a new connection if necessary. If any exceptions are caught, the message is put() back on the queue and the exception is allowed to propagate up through logging.Handler.handleError(), potentially causing this thread to abort. ''' INTERNAL_LOG.debug('Log I/O thread started') while True: record = self.queue.get() if record is None: break jsonrecord = self.jsonify(record) jsonrecord = '%s\n' % jsonrecord try: if self.sock is None: self.connect() self.send(jsonrecord) except Exception: # This exception will be silently ignored and the message # requeued unless self.raiseExceptions=1 self.queue.put(record) self.handleError(record) self.queue.task_done() INTERNAL_LOG.debug('Log I/O thread exited cleanly') def send(self, data): ''' Keep calling SSLSocket.write until the entire message has been sent ''' while len(data) > 0: if self.ssl_ca_file: sent = self.sock.write(data) else: sent = self.sock.send(data) data = data[sent:] self.connect_wait = BACKOFF_INITIAL def handleError(self, record): ''' If an error occurs trying to send the log message, close the connection and delegate the exception handling to the superclass' handleError, which raises the exception (potentially killing the log thread) unless self.raiseExceptions is False. http://hg.python.org/cpython/file/e64d4518b23c/Lib/logging/__init__.py#l797 ''' INTERNAL_LOG.exception('Unable to send log') self.cleanup() self.connect_wait *= BACKOFF_MULTIPLE logging.Handler.handleError(self, record) def cleanup(self): ''' If the socket to the server is still open, close it. Otherwise, do nothing. ''' if self.sock: INTERNAL_LOG.info('Closing socket') self.sock.close() self.sock = None def close(self): ''' Send a sentinel None object to the worker thread, telling it to exit and disconnect from the server. ''' self.queue.put(None) self.cleanup()
class Impinj(object): def __init__(self, dataQ, strayQ, messageQ, shutdownQ, impinjHost, impinjPort, antennaStr, statusCB): self.impinjHost = impinjHost self.impinjPort = impinjPort self.statusCB = statusCB if not antennaStr: self.antennas = [0] else: self.antennas = [int(a) for a in antennaStr.split()] self.tagGroup = None self.tagGroupTimer = None self.dataQ = dataQ # Queue to write tag reads. self.strayQ = strayQ # Queue to write stray reads. self.messageQ = messageQ # Queue to write operational messages. self.shutdownQ = shutdownQ # Queue to listen for shutdown. self.logQ = Queue() self.rospecID = 123 self.readerSocket = None self.timeCorrection = None # Correction between the reader's time and the computer's time. self.connectedAntennas = [] self.antennaReadCount = defaultdict(int) self.lastReadTime = {} self.start() def start(self): # Create a log file name. tNow = getTimeNow() dataDir = os.path.join(HOME_DIR, 'ImpinjData') if not os.path.isdir(dataDir): os.makedirs(dataDir) self.fname = os.path.join( dataDir, tNow.strftime('Impinj-%Y-%m-%d-%H-%M-%S.txt')) # Create a log queue and start a thread to write the log. self.logQ.put('msg', 'Tag ID,Discover Time') self.logFileThread = threading.Thread(target=self.handleLogFile) self.logFileThread.daemon = True self.logFileThread.start() self.keepGoing = True self.tagCount = 0 #------------------------------------------------------------------------- def checkKeepGoing(self): if not self.keepGoing: return False try: # Check the shutdown queue for a message. If there is one, shutdown. d = self.shutdownQ.get(False) self.keepGoing = False return False except Empty: return True def reconnectDelay(self): if self.checkKeepGoing(): time.sleep(ReconnectDelaySeconds) #------------------------------------------------------------------------- def sendCommand(self, message): self.messageQ.put( ('Impinj', '-----------------------------------------------------')) self.messageQ.put(('Impinj', 'Sending Message:\n{}\n'.format(message))) try: message.send(self.readerSocket) except Exception as e: self.messageQ.put(('Impinj', 'Send command fails: {}'.format(e))) return False try: response = WaitForMessage(message.MessageID, self.readerSocket) except Exception as e: self.messageQ.put(('Impinj', 'Get response fails: {}'.format(e))) return False self.messageQ.put( ('Impinj', 'Received Response:\n{}\n'.format(response))) return True, response def sendCommands(self): self.connectedAntennas = [] self.antennaReadCount = defaultdict(int) self.messageQ.put( ('Impinj', 'Connected to: ({}:{})'.format(self.impinjHost, self.impinjPort))) self.messageQ.put( ('Impinj', 'Waiting for READER_EVENT_NOTIFICATION...')) response = UnpackMessageFromSocket(self.readerSocket) self.messageQ.put( ('Impinj', '\nReceived Response:\n{}\n'.format(response))) # Compute a correction between the reader's time and the computer's time. readerTime = response.getFirstParameterByClass( UTCTimestamp_Parameter).Microseconds readerTime = datetime.datetime.utcfromtimestamp(readerTime / 1000000.0) self.timeCorrection = getTimeNow() - readerTime self.messageQ.put( ('Impinj', '\nReader time is {} seconds different from computer time\n'. format(self.timeCorrection.total_seconds()))) # Reset to factory defaults. success, response = self.sendCommand( SET_READER_CONFIG_Message(ResetToFactoryDefault=True)) if not success: return False # Get the connected antennas. success, response = self.sendCommand( GET_READER_CONFIG_Message( RequestedData=GetReaderConfigRequestedData.AntennaProperties)) if success: self.connectedAntennas = [ p.AntennaID for p in response.Parameters if isinstance(p, AntennaProperties_Parameter) and p.AntennaConnected and p.AntennaID <= 4 ] # Configure a periodic Keepalive message. # Change receiver sensitivity (if specified). This value is reader dependent. receiverSensitivityParameter = [] if ReceiverSensitivity is not None: receiverSensitivityParameter.append( RFReceiver_Parameter(ReceiverSensitivity=ReceiverSensitivity)) # Change transmit power (if specified). This value is reader dependent. transmitPowerParameter = [] if TransmitPower is not None: transmitPowerParameter.append( RFTransmitter_Parameter( HopTableID=1, ChannelIndex=0, TransmitPower=TransmitPower, )) # Change Inventory Control (if specified). inventoryCommandParameter = [] if any(v is not None for v in [InventorySession, TagPopulation, TagTransitTime]): inventoryCommandParameter.append( C1G2InventoryCommand_Parameter(Parameters=[ C1G2SingulationControl_Parameter( Session=InventorySession or 0, TagPopulation=TagPopulation or TagPopulationDefault, TagTransitTime=(TagTransitTime or 3) * 1000, ), ], )) success, response = self.sendCommand( SET_READER_CONFIG_Message(Parameters=[ AntennaConfiguration_Parameter( AntennaID=0, Parameters=receiverSensitivityParameter + transmitPowerParameter + inventoryCommandParameter, ), KeepaliveSpec_Parameter( KeepaliveTriggerType=KeepaliveTriggerType.Periodic, PeriodicTriggerValue=int(KeepaliveSeconds * 1000), ), ], ), ) if not success: return False # Disable all rospecs in the reader. success, response = self.sendCommand( DISABLE_ROSPEC_Message(ROSpecID=0)) if not success: return False # Delete our old rospec. success, response = self.sendCommand( DELETE_ROSPEC_Message(ROSpecID=self.rospecID)) if not success: return False # Get the C1G2UHFRFModeTable and extract available mode identifiers. modeIdentifiers = None maxNumberOfAntennaSupported = 4 try: success, response = self.sendCommand( GET_READER_CAPABILITIES_Message( RequestedData=GetReaderCapabilitiesRequestedData.All)) if success: modeIdentifiers = [ e.ModeIdentifier for e in response.getFirstParameterByClass( C1G2UHFRFModeTable_Parameter).Parameters ] gdc = response.getFirstParameterByClass( GeneralDeviceCapabilities_Parameter) maxNumberOfAntennaSupported = gdc.MaxNumberOfAntennaSupported else: self.messageQ.put(('Impinj', 'GET_READER_CAPABILITIES fails.')) except Exception as e: self.messageQ.put( ('Impinj', 'GET_READER_CAPABILITIES Exception: {}:\n{}'.format( e, traceback.format_exc()))) # Configure our new rospec. if ProcessingMethod == FirstReadMethod: cmd = GetBasicAddRospecMessage(ROSpecID=self.rospecID, antennas=self.antennas) else: cmd = GetAddRospecRSSIMessage( ROSpecID=self.rospecID, antennas=self.antennas, modeIdentifiers=modeIdentifiers, maxNumberOfAntennaSupported=maxNumberOfAntennaSupported) success, response = self.sendCommand(cmd) if not success: return False # Enable our new rospec. success, response = self.sendCommand( ENABLE_ROSPEC_Message(ROSpecID=self.rospecID)) if not success: return False success = (success and isinstance(response, ENABLE_ROSPEC_RESPONSE_Message) and response.success()) return success def reportTag(self, tagID, discoveryTime, sampleSize=1, antennaID=0, quadReg=False): lrt = self.lastReadTime.get(tagID, tOld) if discoveryTime > lrt: self.lastReadTime[tagID] = discoveryTime if (discoveryTime - lrt).total_seconds() < RepeatSeconds: self.messageQ.put(( 'Impinj', 'Received {}. tag={} Skipped (<{} secs ago). {}'.format( self.tagCount, tagID, RepeatSeconds, discoveryTime.strftime('%H:%M:%S.%f')), self.antennaReadCount, )) return False self.dataQ.put((tagID, discoveryTime)) self.logQ.put(('log', '{},{}'.format( tagID, discoveryTime.strftime('%a %b %d %H:%M:%S.%f %Z %Y-%m-%d'), ))) self.messageQ.put(( 'Impinj', '{} {}. {} - {}{}{}'.format( 'QuadReg' if quadReg else 'FirstRead', self.tagCount, tagID, discoveryTime.strftime('%H:%M:%S.%f'), ' samples={}'.format(sampleSize) if sampleSize > 1 else '', ' antennaID={}'.format(antennaID) if antennaID else '', ), self.antennaReadCount, )) Bell() return True def handleTagGroup(self): if not self.tagGroup: return reads, strays = self.tagGroup.getReadsStrays( method=ProcessingMethod, antennaChoice=AntennaChoice) for tagID, discoveryTime, sampleSize, antennaID in reads: self.reportTag(tagID, discoveryTime, sampleSize, antennaID, True) self.strayQ.put(('strays', strays)) self.tagGroupTimer = threading.Timer(1.0, self.handleTagGroup) self.tagGroupTimer.start() def handleLogFile(self): while True: msg = self.logQ.get() self.logQ.task_done() if msg[0] == 'shutdown': return try: pf = open(self.fname, 'a') except: continue pf.write(msg[1] if msg[1].endswith('\n') else msg[1] + '\n') while True: try: msg = self.logQ.get(False) except Empty: break self.logQ.task_done() if msg[0] == 'shutdown': return pf.write(msg[1] if msg[1].endswith('\n') else msg[1] + '\n') pf.close() time.sleep(0.1) def runServer(self): self.messageQ.put(('BackupFile', self.fname)) self.messageQ.put( ('Impinj', '*****************************************')) self.messageQ.put( ('Impinj', 'Reader Server Started: ({}:{})'.format(self.impinjHost, self.impinjPort))) # Create an old default time for last tag read. tOld = getTimeNow() - datetime.timedelta(days=100) utcfromtimestamp = datetime.datetime.utcfromtimestamp while self.checkKeepGoing(): #------------------------------------------------------------ # Connect Mode. # # Create a socket to connect to the reader. self.readerSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.readerSocket.settimeout(ConnectionTimeoutSeconds) self.messageQ.put(('Impinj', 'state', False)) self.messageQ.put(('Impinj', '')) self.messageQ.put( ('Impinj', 'Trying to Connect to Reader: ({}:{})...'.format( self.impinjHost, self.impinjPort))) self.messageQ.put( ('Impinj', 'ConnectionTimeout={:.2f} seconds'.format( ConnectionTimeoutSeconds))) try: self.readerSocket.connect((self.impinjHost, self.impinjPort)) except Exception as e: self.messageQ.put( ('Impinj', 'Reader Connection Failed: {}'.format(e))) self.readerSocket.close() self.messageQ.put( ('Impinj', 'Attempting Reconnect in {} seconds...'.format( ReconnectDelaySeconds))) self.reconnectDelay() continue self.messageQ.put(('Impinj', 'state', True)) try: success = self.sendCommands() except Exception as e: self.messageQ.put( ('Impinj', 'Send Command Error={}'.format(e))) success = False if not success: self.messageQ.put(('Impinj', 'Reader Initialization Failed.')) self.messageQ.put(('Impinj', 'Disconnecting Reader.')) self.messageQ.put(('Impinj', 'state', False)) self.readerSocket.close() self.messageQ.put( ('Impinj', 'Attempting Reconnect in {} seconds...'.format( ReconnectDelaySeconds))) self.reconnectDelay() self.statusCB() continue self.statusCB( connectedAntennas=self.connectedAntennas, timeCorrection=self.timeCorrection, ) self.tagGroup = TagGroup() self.handleTagGroup() tUpdateLast = tKeepaliveLast = getTimeNow() self.tagCount = 0 lastDiscoveryTime = None while self.checkKeepGoing(): #------------------------------------------------------------ # Read Mode. # try: response = UnpackMessageFromSocket(self.readerSocket) except socket.timeout: t = getTimeNow() if (t - tKeepaliveLast ).total_seconds() > KeepaliveSeconds * 2: self.messageQ.put( ('Impinj', 'Reader Connection Lost (missing Keepalive).')) self.readerSocket.close() self.messageQ.put( ('Impinj', 'Attempting Reconnect...')) break if (t - tUpdateLast ).total_seconds() >= ReaderUpdateMessageSeconds: self.messageQ.put( ('Impinj', 'Listening for Impinj reader data...')) tUpdateLast = t continue if isinstance(response, KEEPALIVE_Message): # Respond to the KEEP_ALIVE message with KEEP_ALIVE_ACK. try: KEEPALIVE_ACK_Message().send(self.readerSocket) except socket.timeout: self.messageQ.put(( 'Impinj', 'Reader Connection Lost (Keepalive_Ack timeout).')) self.readerSocket.close() self.messageQ.put( ('Impinj', 'Attempting Reconnect...')) break tKeepaliveLast = getTimeNow() continue if not isinstance(response, RO_ACCESS_REPORT_Message): if not isinstance(response, READER_EVENT_NOTIFICATION_Message): self.messageQ.put(('Impinj', 'Skipping: {}'.format( response.__class__.__name__))) continue try: discoveryTime = utcfromtimestamp(tag['Timestamp'] / 1000000.0) if ImpinjDebug and lastDiscoveryTime is not None: print('{} \r'.format( (discoveryTime - lastDiscoveryTime).total_seconds())) lastDiscoveryTime = discoveryTime except: pass for tag in response.getTagData(): self.tagCount += 1 antennaID = tag['AntennaID'] try: self.antennaReadCount[antennaID] += 1 except Exception as e: self.messageQ.put( ('Impinj', 'Received {}. Missing AntennaID.'.format( self.tagCount))) try: tagID = tag['EPC'] except Exception as e: self.messageQ.put( ('Impinj', 'Received {}. Skipping: missing tagID.'.format( self.tagCount))) continue try: tagID = HexFormatToStr(tagID) except Exception as e: self.messageQ.put(( 'Impinj', 'Received {}. Skipping: HexFormatToStr fails. Error={}' .format(self.tagCount, e))) continue try: discoveryTime = tag[ 'Timestamp'] # In microseconds since Jan 1, 1970 except Exception as e: self.messageQ.put(( 'Impinj', 'Received {}. Skipping: Missing Timestamp'.format( self.tagCount))) continue peakRSSI = tag.get('PeakRSSI', None) # -127..127 in db. # Convert discoveryTime to Python format and correct for reader time difference. discoveryTime = utcfromtimestamp( discoveryTime / 1000000.0) + self.timeCorrection if peakRSSI is not None: self.tagGroup.add(antennaID, tagID, discoveryTime, peakRSSI) else: self.reportTag(tagID, discoveryTime, antennaID=antennaID) # Cleanup. if self.readerSocket: try: response = self.sendCommand(CLOSE_CONNECTION_Message()) except socket.timeout: pass self.readerSocket.close() self.readerSocket = None self.logQ.put(('shutdown', )) self.logFileThread.join() if self.tagGroupTimer: self.tagGroupTimer.cancel() return True def purgeDataQ(self): while True: try: d = self.dataQ.get(False) except Empty: break
class MyApp(object): instances = 0 def __init__(self, filename=None): MyApp.instances += 1 self.file_dlg = None self.info = None self.page_id = 0 self.keep_running = True self.queue = Queue() thread = Thread(target=self.worker_loop) thread.daemon = True thread.start() builder = Gtk.Builder() builder.add_from_file(GLADE_FN) builder.connect_signals(self) self.window = builder.get_object("main_win") self.header = builder.get_object("header") self.body = builder.get_object("body") self.toc_store = builder.get_object("toc_store") self.toc_tree = builder.get_object("toc_tree") self.window.show() if filename is not None: self.open(filename) def on_info_btn_clicked(self, w): self.goto_page(0) def on_previous_btn_clicked(self, w): self.goto_page(max(0, self.page_id - 1)) def on_next_btn_clicked(self, w): self.goto_page(self.page_id + 1) def on_search_entry_activate(self, w): text = w.get_text() page_id = try_int(text) if page_id is not None: self.goto_page(page_id) def worker_loop(self): while self.keep_running: try: a = self.queue.get(timeout=10) except Empty: continue cb_name, kwargs = a cb = getattr(self, cb_name) if not cb: self.queue.task_done() continue try: cb(**kwargs) except Exception as e: logger.error("ERROR: %r", e) self.queue.task_done() logger.info("worker thread exited") def open(self, filename): self.filename = filename cols = get_table_col(filename, 'Main') self.db = db = pyodbc.connect( tob('DRIVER=libmdbodbc.so;DBQ={}'.format(filename)), readonly=True, ansi=True, unicode_results=False, ) cursor = db.cursor() cursor.execute(u'SELECT {} FROM Main'.format(','.join(cols))) self.info = row_to_dict(cursor.fetchone(), cols) self.header.set_title(self.info['Bk']) self.goto_page(0) #cols = cursor.columns('Main') # does not work self.id = int(self.info['BkId']) cursor = db.cursor() tbl_toc = 't{}'.format(self.info['BkId']) cols = get_table_col(filename, tbl_toc) cursor.execute(u'SELECT {} FROM {}'.format(','.join(cols), tbl_toc)) rows = [row_to_dict(row, cols) for row in cursor.fetchall()] rows.sort(key=lambda r: (r['id'], r['sub'])) def cb(r): for row in r: self.toc_store.append(None, ( row['tit'], row['lvl'], row['sub'], row['id'], )) # it's a store, not UI, so we might be able to edit it directly # cb(rows) # if not then it's added like this GLib.idle_add(cb, rows) def goto_page(self, page_id, move_toc=False): if self.info is None: return self.page_id = page_id if page_id == 0: text = self.info['Betaka'] else: tbl_body = 'b{}'.format(self.info['BkId']) cols = get_table_col(self.filename, tbl_body) cursor = self.db.cursor() cursor.execute(u'SELECT {} FROM {} WHERE id={}'.format( ','.join(cols), tbl_body, page_id)) self.page = row_to_dict(cursor.fetchone(), cols) text = self.page['nass'] GLib.idle_add(lambda: self.body.get_buffer().set_text(text)) def on_window_destroy(self, w): self.keep_running = False MyApp.instances -= 1 logger.info("running instances = %r", MyApp.instances) if MyApp.instances == 0: Gtk.main_quit() def on_toc_tree_selection_changed(self, w): s, i = w.get_selected() # can be accessed in many ways row=tuple(s[i]) or id=s[i][3] or id=s.get_value(i, 3) self.queue.put(( 'goto_page', { 'page_id': s[i][3] }, )) def on_open_btn_clicked(self, w): filename = get_filename(self.window) if filename: if self.info is None: self.queue.put(( 'open', { 'filename': filename }, )) else: MyApp(filename)
class SocketServer(object): """ZMQ-based socket server for sending and receiving messages from the host PC. Because of the weird way in which PyEPL handles events, we can't run this as its own thread, but instead have to poll for events in the general PyEPL machinery. In the future, we should clean up PyEPL entirely so that it does not block other threads (amongst other reasons). :param zmq.Context ctx: """ def __init__(self, ctx=None): self.ctx = ctx or zmq.Context() self._handlers = [] self.sock = self.ctx.socket(zmq.PAIR) self._bound = False self.poller = zmq.Poller() self.poller.register(self.sock, zmq.POLLIN) # Outgoing message queue self._out_queue = Queue() # time of last sent heartbeat message self._last_heartbeat = 0. # Logging of sent and received messages. self.logger = create_logger("network") def join(self): """Block until all outgoing messages have been processed.""" self.logger.warning("Joining doesn't work yet; doing nothing...") # self._out_queue.join() def bind(self, address="tcp://*:8889"): """Bind the socket to start listening for connections. :param str address: ZMQ address string """ self.sock.bind(address) self._bound = True def register_handler(self, func): """Register a message handler. :param callable func: Handler function which takes the message as its only argument. """ self.logger.debug("Adding handler: %s", func.__name__) self._handlers.append(func) def enqueue_message(self, msg): """Submit a new outgoing message to the queue.""" self._out_queue.put_nowait(msg) def send(self, msg): """Immediately transmit a message to the host PC. It is advisable to not call this method directly in most cases, but rather enqueue a message to be sent via :meth:`enqueue_message`. :param RAMMessage msg: Message to send. """ out = msg.jsonize() try: self.log_message(msg, incoming=False) self.sock.send(out, zmq.NOBLOCK) except: self.logger.error("Sending failed!") def send_heartbeat(self): """Convenience method to send a heartbeat message to the host PC.""" if time.time() - self._last_heartbeat >= 1.0: self.send(HeartbeatMessage()) self._last_heartbeat = time.time() def log_message(self, message, incoming=True): """Log a message to the log file.""" if not incoming: message = message.to_dict() message["in_or_out"] = "in" if incoming else "out" self.logger.info("%s", json.dumps(message)) def handle_incoming(self): events = self.poller.poll(1) if self.sock in dict(events): try: msg = self.sock.recv_json() self.log_message(msg, incoming=True) except: self.logger.error("Unable to decode JSON.", exc_info=True) return for handler in self._handlers: try: handler(msg) except: self.logger.error("Error handling message", exc_info=True) continue def handle_outgoing(self): try: while not self._out_queue.empty(): msg = self._out_queue.get_nowait() self.send(msg) self._out_queue.task_done( ) # so we can join the queue elsewhere except: self.logger.error("Error in outgoing message processing", exc_info=True) def update(self): """Call periodically to check for incoming messages and/or send messages in the outgoing queue. """ self.handle_incoming() self.handle_outgoing()
class GraphiteReporter(threading.Thread): """A graphite reporter thread.""" def __init__(self, host, port, maxQueueSize=10000): """Connect to a Graphite server on host:port.""" threading.Thread.__init__(self) self.host, self.port = host, port self.sock = None self.queue = Queue() self.maxQueueSize = maxQueueSize self.daemon = True def run(self): """Run the thread.""" while True: try: try: name, value, valueType, stamp = self.queue.get() except TypeError: break self.log(name, value, valueType, stamp) finally: self.queue.task_done() def connect(self): """Connects to the Graphite server if not already connected.""" if self.sock is not None: return backoff = 0.01 while True: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(5) sock.connect((self.host, self.port)) self.sock = sock return except socket.error: time.sleep(random.uniform(0, 2.0 * backoff)) backoff = min(backoff * 2.0, 5.0) def disconnect(self): """Disconnect from the Graphite server if connected.""" if self.sock is not None: try: self.sock.close() except socket.error: pass finally: self.sock = None def _sendMsg(self, msg): """Send a line to graphite. Retry with exponential backoff.""" if not self.sock: self.connect() backoff = 0.001 while True: try: self.sock.sendall(msg) break except socket.error: logging.warning('Graphite connection error', exc_info=True) self.disconnect() time.sleep(random.uniform(0, 2.0 * backoff)) backoff = min(backoff * 2.0, 5.0) self.connect() def _sanitizeName(self, name): """Sanitize a metric name.""" return name.replace(' ', '-') def log(self, name, value, valueType=None, stamp=None): """Log a named numeric value. The value type may be 'value', 'count', or None.""" if type(value) == float: form = "%s%s %2.2f %d\n" else: form = "%s%s %s %d\n" if valueType is not None and len( valueType) > 0 and valueType[0] != '.': valueType = '.' + valueType if not stamp: stamp = time.time() self._sendMsg( form % (self._sanitizeName(name), valueType or '', value, stamp)) def enqueue(self, name, value, valueType=None, stamp=None): """Enqueue a call to log.""" # If queue is too large, refuse to log. if self.maxQueueSize and self.queue.qsize() > self.maxQueueSize: return # Stick arguments into the queue self.queue.put((name, value, valueType, stamp)) def flush(self): """Block until all stats have been sent to Graphite.""" self.queue.join() def shutdown(self): """Shut down the background thread.""" self.queue.put(None) self.flush()
class Session: """A session with a NETCONF server This class is a context manager, and should always be either used with a ``with`` statement or the :meth:`close` method should be called manually when the object is no longer required. :ivar server_capabilities: The list of capabilities parsed from the server's ``<hello>`` :ivar client_capabilities: The list of capabilities parsed from the client's ``<hello>`` """ def __init__(self, sock): self.sock = sock self.mode = "1.0" self.send_msg(DEFAULT_HELLO) self.client_hello = DEFAULT_HELLO self.parser = parse_messages(sock, self.mode) # First message will be the server hello self.server_hello = next(self.parser) server_ele = etree.fromstring(self.server_hello) self.session_id = int( server_ele.xpath("/nc:hello/nc:session-id", namespaces=NAMESPACES)[0].text) self.server_capabilities = capabilities_from_hello(server_ele) client_ele = etree.fromstring(self.client_hello) self.client_capabilities = capabilities_from_hello(client_ele) if (CAP_NETCONF_11 in self.client_capabilities and CAP_NETCONF_11 in self.server_capabilities): self.mode = "1.1" self.unknown_recvq = Queue() self.notifications = Queue() self.rpc_reply_futures = Queue() self.thread = Thread(target=self._recv_loop) self.thread.daemon = True self.thread.start() def __enter__(self): return self def __exit__(self, _, __, ___): self.close() def close(self): """Closes any associated sockets and frees any other associated resources""" try: self.sock.close() except Exception: pass try: while True: f = self.rpc_reply_futures.get(block=False) f.set_exception(SessionClosedException()) self.rpc_reply_futures.task_done() except Empty: pass def send_msg(self, msg): """Sends a raw byte string to the server :param bytes msg: The byte string to send """ logger.debug("Sending message on session %s", msg) if self.mode == "1.0": self.sock.sendall(msg + b"]]>]]>") elif self.mode == "1.1": self.sock.sendall(frame_message_11(msg)) def send_rpc(self, rpc): """Sends a raw RPC to the server :param bytes rpc: The RPC to send :rtype: :class:`concurrent.futures.Future` with a result type of tuple(:class:`bytes`, :class:`lxml.Element`) """ f = Future() self.rpc_reply_futures.put(f) self.send_msg(rpc) return f def _recv_loop(self): while True: try: msg = self.parser.send(self.mode) except Exception as e: logger.info("Stopping recv thread due to exception %s", e) return ele = etree.fromstring(msg) if ele.xpath("/nc:rpc-reply", namespaces=NAMESPACES): try: f = self.rpc_reply_futures.get(block=False) if ele.xpath("/nc:rpc-reply/nc:rpc-error", namespaces=NAMESPACES): f.set_exception(RpcError(msg, ele)) else: f.set_result((msg, ele)) self.rpc_reply_futures.task_done() msg = None except Empty: logger.warning( "An <rpc-reply> was received " "with no corresponding handler: %s", msg, ) elif ele.xpath("/notif:notification", namespaces=NAMESPACES): self.notifications.put((msg, ele)) msg = None if msg is not None: self.unknown_recvq.put((msg, ele))
class Impinj( object ): def __init__( self, dataQ, strayQ, messageQ, shutdownQ, impinjHost, impinjPort, antennaStr, statusCB ): self.impinjHost = impinjHost self.impinjPort = impinjPort self.statusCB = statusCB if not antennaStr: self.antennas = [0] else: self.antennas = [int(a) for a in antennaStr.split()] self.tagGroup = None self.tagGroupTimer = None self.dataQ = dataQ # Queue to write tag reads. self.strayQ = strayQ # Queue to write stray reads. self.messageQ = messageQ # Queue to write operational messages. self.shutdownQ = shutdownQ # Queue to listen for shutdown. self.logQ = Queue() self.rospecID = 123 self.readerSocket = None self.timeCorrection = None # Correction between the reader's time and the computer's time. self.connectedAntennas = [] self.antennaReadCount = defaultdict(int) self.lastReadTime = {} self.start() def start( self ): # Create a log file name. tNow = getTimeNow() dataDir = os.path.join( HOME_DIR, 'ImpinjData' ) if not os.path.isdir( dataDir ): os.makedirs( dataDir ) self.fname = os.path.join( dataDir, tNow.strftime('Impinj-%Y-%m-%d-%H-%M-%S.txt') ) # Create a log queue and start a thread to write the log. self.logQ.put( 'msg', 'Tag ID,Discover Time' ) self.logFileThread = threading.Thread( target=self.handleLogFile ) self.logFileThread.daemon = True self.logFileThread.start() self.keepGoing = True self.tagCount = 0 #------------------------------------------------------------------------- def checkKeepGoing( self ): if not self.keepGoing: return False try: # Check the shutdown queue for a message. If there is one, shutdown. d = self.shutdownQ.get( False ) self.keepGoing = False return False except Empty: return True def reconnectDelay( self ): if self.checkKeepGoing(): time.sleep( ReconnectDelaySeconds ) #------------------------------------------------------------------------- def sendCommand( self, message ): self.messageQ.put( ('Impinj', '-----------------------------------------------------') ) self.messageQ.put( ('Impinj', 'Sending Message:\n{}\n'.format(message)) ) try: message.send( self.readerSocket ) except Exception as e: self.messageQ.put( ('Impinj', 'Send command fails: {}'.format(e)) ) return False try: response = WaitForMessage( message.MessageID, self.readerSocket ) except Exception as e: self.messageQ.put( ('Impinj', 'Get response fails: {}'.format(e)) ) return False self.messageQ.put( ('Impinj', 'Received Response:\n{}\n'.format(response)) ) return True, response def sendCommands( self ): self.connectedAntennas = [] self.antennaReadCount = defaultdict(int) self.messageQ.put( ('Impinj', 'Connected to: ({}:{})'.format(self.impinjHost, self.impinjPort) ) ) self.messageQ.put( ('Impinj', 'Waiting for READER_EVENT_NOTIFICATION...') ) response = UnpackMessageFromSocket( self.readerSocket ) self.messageQ.put( ('Impinj', '\nReceived Response:\n{}\n'.format(response)) ) # Compute a correction between the reader's time and the computer's time. readerTime = response.getFirstParameterByClass(UTCTimestamp_Parameter).Microseconds readerTime = datetime.datetime.utcfromtimestamp( readerTime / 1000000.0 ) self.timeCorrection = getTimeNow() - readerTime self.messageQ.put( ('Impinj', '\nReader time is {} seconds different from computer time\n'.format(self.timeCorrection.total_seconds())) ) # Reset to factory defaults. success, response = self.sendCommand( SET_READER_CONFIG_Message(ResetToFactoryDefault = True) ) if not success: return False # Get the connected antennas. success, response = self.sendCommand( GET_READER_CONFIG_Message(RequestedData=GetReaderConfigRequestedData.AntennaProperties) ) if success: self.connectedAntennas = [p.AntennaID for p in response.Parameters if isinstance(p, AntennaProperties_Parameter) and p.AntennaConnected and p.AntennaID <= 4] # Configure a periodic Keepalive message. # Change receiver sensitivity (if specified). This value is reader dependent. receiverSensitivityParameter = [] if ReceiverSensitivity is not None: receiverSensitivityParameter.append( RFReceiver_Parameter( ReceiverSensitivity = ReceiverSensitivity ) ) # Change transmit power (if specified). This value is reader dependent. transmitPowerParameter = [] if TransmitPower is not None: transmitPowerParameter.append( RFTransmitter_Parameter( HopTableID = 1, ChannelIndex = 0, TransmitPower = TransmitPower, ) ) # Change Inventory Control (if specified). inventoryCommandParameter = [] if any(v is not None for v in [InventorySession, TagPopulation, TagTransitTime]): inventoryCommandParameter.append( C1G2InventoryCommand_Parameter( Parameters = [ C1G2SingulationControl_Parameter( Session = InventorySession or 0, TagPopulation = TagPopulation or TagPopulationDefault, TagTransitTime = (TagTransitTime or 3)*1000, ), ], ) ) success, response = self.sendCommand( SET_READER_CONFIG_Message( Parameters = [ AntennaConfiguration_Parameter( AntennaID = 0, Parameters = receiverSensitivityParameter + transmitPowerParameter + inventoryCommandParameter, ), KeepaliveSpec_Parameter( KeepaliveTriggerType = KeepaliveTriggerType.Periodic, PeriodicTriggerValue = int(KeepaliveSeconds*1000), ), ], ), ) if not success: return False # Disable all rospecs in the reader. success, response = self.sendCommand( DISABLE_ROSPEC_Message(ROSpecID = 0) ) if not success: return False # Delete our old rospec. success, response = self.sendCommand( DELETE_ROSPEC_Message(ROSpecID = self.rospecID) ) if not success: return False # Get the C1G2UHFRFModeTable and extract available mode identifiers. modeIdentifiers = None maxNumberOfAntennaSupported = 4 try: success, response = self.sendCommand(GET_READER_CAPABILITIES_Message(RequestedData = GetReaderCapabilitiesRequestedData.All)) if success: modeIdentifiers = [e.ModeIdentifier for e in response.getFirstParameterByClass(C1G2UHFRFModeTable_Parameter).Parameters] gdc = response.getFirstParameterByClass(GeneralDeviceCapabilities_Parameter) maxNumberOfAntennaSupported = gdc.MaxNumberOfAntennaSupported else: self.messageQ.put( ('Impinj', 'GET_READER_CAPABILITIES fails.') ) except Exception as e: self.messageQ.put( ('Impinj', 'GET_READER_CAPABILITIES Exception: {}:\n{}'.format(e, traceback.format_exc())) ) # Configure our new rospec. if ProcessingMethod == FirstReadMethod: cmd = GetBasicAddRospecMessage(ROSpecID = self.rospecID, antennas = self.antennas) else: cmd = GetAddRospecRSSIMessage( ROSpecID = self.rospecID, antennas = self.antennas, modeIdentifiers=modeIdentifiers, maxNumberOfAntennaSupported=maxNumberOfAntennaSupported ) success, response = self.sendCommand(cmd) if not success: return False # Enable our new rospec. success, response = self.sendCommand( ENABLE_ROSPEC_Message(ROSpecID = self.rospecID) ) if not success: return False success = (success and isinstance(response, ENABLE_ROSPEC_RESPONSE_Message) and response.success()) return success def reportTag( self, tagID, discoveryTime, sampleSize=1, antennaID=0, quadReg=False ): lrt = self.lastReadTime.get(tagID, tOld) if discoveryTime > lrt: self.lastReadTime[tagID] = discoveryTime if (discoveryTime - lrt).total_seconds() < RepeatSeconds: self.messageQ.put( ( 'Impinj', 'Received {}. tag={} Skipped (<{} secs ago). {}'.format(self.tagCount, tagID, RepeatSeconds, discoveryTime.strftime('%H:%M:%S.%f')), self.antennaReadCount, ) ) return False self.dataQ.put( (tagID, discoveryTime) ) self.logQ.put( ( 'log', '{},{}'.format( tagID, discoveryTime.strftime('%a %b %d %H:%M:%S.%f %Z %Y-%m-%d'), ) ) ) self.messageQ.put( ( 'Impinj', '{} {}. {} - {}{}{}'.format( 'QuadReg' if quadReg else 'FirstRead', self.tagCount, tagID, discoveryTime.strftime('%H:%M:%S.%f'), ' samples={}'.format(sampleSize) if sampleSize > 1 else '', ' antennaID={}'.format(antennaID) if antennaID else '', ), self.antennaReadCount, ) ) Bell() return True def handleTagGroup( self ): if not self.tagGroup: return reads, strays = self.tagGroup.getReadsStrays( method=ProcessingMethod, antennaChoice=AntennaChoice ) for tagID, discoveryTime, sampleSize, antennaID in reads: self.reportTag( tagID, discoveryTime, sampleSize, antennaID, True ) self.strayQ.put( ('strays', strays) ) self.tagGroupTimer = threading.Timer( 1.0, self.handleTagGroup ) self.tagGroupTimer.start() def handleLogFile( self ): while 1: msg = self.logQ.get() self.logQ.task_done() if msg[0] == 'shutdown': return try: pf = io.open( self.fname, 'a' ) except: continue pf.write( msg[1] if msg[1].endswith('\n') else msg[1] + '\n' ) while 1: try: msg = self.logQ.get( False ) except Empty: break self.logQ.task_done() if msg[0] == 'shutdown': return pf.write( msg[1] if msg[1].endswith('\n') else msg[1] + '\n' ) pf.close() time.sleep( 0.1 ) def runServer( self ): self.messageQ.put( ('BackupFile', self.fname) ) self.messageQ.put( ('Impinj', '*****************************************' ) ) self.messageQ.put( ('Impinj', 'Reader Server Started: ({}:{})'.format(self.impinjHost, self.impinjPort) ) ) # Create an old default time for last tag read. tOld = getTimeNow() - datetime.timedelta( days = 100 ) utcfromtimestamp = datetime.datetime.utcfromtimestamp while self.checkKeepGoing(): #------------------------------------------------------------ # Connect Mode. # # Create a socket to connect to the reader. self.readerSocket = socket.socket( socket.AF_INET, socket.SOCK_STREAM ) self.readerSocket.settimeout( ConnectionTimeoutSeconds ) self.messageQ.put( ('Impinj', 'state', False) ) self.messageQ.put( ('Impinj', '') ) self.messageQ.put( ('Impinj', 'Trying to Connect to Reader: ({}:{})...'.format(self.impinjHost, self.impinjPort) ) ) self.messageQ.put( ('Impinj', 'ConnectionTimeout={:.2f} seconds'.format(ConnectionTimeoutSeconds) ) ) try: self.readerSocket.connect( (self.impinjHost, self.impinjPort) ) except Exception as e: self.messageQ.put( ('Impinj', 'Reader Connection Failed: {}'.format(e) ) ) self.readerSocket.close() self.messageQ.put( ('Impinj', 'Attempting Reconnect in {} seconds...'.format(ReconnectDelaySeconds)) ) self.reconnectDelay() continue self.messageQ.put( ('Impinj', 'state', True) ) try: success = self.sendCommands() except Exception as e: self.messageQ.put( ('Impinj', 'Send Command Error={}'.format(e)) ) success = False if not success: self.messageQ.put( ('Impinj', 'Reader Initialization Failed.') ) self.messageQ.put( ('Impinj', 'Disconnecting Reader.' ) ) self.messageQ.put( ('Impinj', 'state', False) ) self.readerSocket.close() self.messageQ.put( ('Impinj', 'Attempting Reconnect in {} seconds...'.format(ReconnectDelaySeconds)) ) self.reconnectDelay() self.statusCB() continue self.statusCB( connectedAntennas = self.connectedAntennas, timeCorrection = self.timeCorrection, ) self.tagGroup = TagGroup() self.handleTagGroup() tUpdateLast = tKeepaliveLast = getTimeNow() self.tagCount = 0 lastDiscoveryTime = None while self.checkKeepGoing(): #------------------------------------------------------------ # Read Mode. # try: response = UnpackMessageFromSocket( self.readerSocket ) except socket.timeout: t = getTimeNow() if (t - tKeepaliveLast).total_seconds() > KeepaliveSeconds * 2: self.messageQ.put( ('Impinj', 'Reader Connection Lost (missing Keepalive).') ) self.readerSocket.close() self.messageQ.put( ('Impinj', 'Attempting Reconnect...') ) break if (t - tUpdateLast).total_seconds() >= ReaderUpdateMessageSeconds: self.messageQ.put( ('Impinj', 'Listening for Impinj reader data...') ) tUpdateLast = t continue if isinstance(response, KEEPALIVE_Message): # Respond to the KEEP_ALIVE message with KEEP_ALIVE_ACK. try: KEEPALIVE_ACK_Message().send( self.readerSocket ) except socket.timeout: self.messageQ.put( ('Impinj', 'Reader Connection Lost (Keepalive_Ack timeout).') ) self.readerSocket.close() self.messageQ.put( ('Impinj', 'Attempting Reconnect...') ) break tKeepaliveLast = getTimeNow() continue if not isinstance(response, RO_ACCESS_REPORT_Message): if not isinstance(response, READER_EVENT_NOTIFICATION_Message): self.messageQ.put( ('Impinj', 'Skipping: {}'.format(response.__class__.__name__)) ) continue try: discoveryTime = utcfromtimestamp( tag['Timestamp'] / 1000000.0 ) if ImpinjDebug and lastDiscoveryTime is not None: six.print_( '{} \r'.format( (discoveryTime - lastDiscoveryTime).total_seconds() ) ) lastDiscoveryTime = discoveryTime except: pass for tag in response.getTagData(): self.tagCount += 1 antennaID = tag['AntennaID'] try: self.antennaReadCount[antennaID] += 1 except Exception as e: self.messageQ.put( ('Impinj', 'Received {}. Missing AntennaID.'.format(self.tagCount)) ) try: tagID = tag['EPC'] except Exception as e: self.messageQ.put( ('Impinj', 'Received {}. Skipping: missing tagID.'.format(self.tagCount)) ) continue try: tagID = HexFormatToStr( tagID ) except Exception as e: self.messageQ.put( ('Impinj', 'Received {}. Skipping: HexFormatToStr fails. Error={}'.format(self.tagCount, e)) ) continue try: discoveryTime = tag['Timestamp'] # In microseconds since Jan 1, 1970 except Exception as e: self.messageQ.put( ('Impinj', 'Received {}. Skipping: Missing Timestamp'.format(self.tagCount)) ) continue peakRSSI = tag.get('PeakRSSI', None) # -127..127 in db. # Convert discoveryTime to Python format and correct for reader time difference. discoveryTime = utcfromtimestamp( discoveryTime / 1000000.0 ) + self.timeCorrection if peakRSSI is not None: self.tagGroup.add( antennaID, tagID, discoveryTime, peakRSSI ) else: self.reportTag( tagID, discoveryTime, antennaID=antennaID ) # Cleanup. if self.readerSocket: try: response = self.sendCommand( CLOSE_CONNECTION_Message() ) except socket.timeout: pass self.readerSocket.close() self.readerSocket = None self.logQ.put( ('shutdown',) ) self.logFileThread.join() if self.tagGroupTimer: self.tagGroupTimer.cancel() return True def purgeDataQ( self ): while 1: try: d = self.dataQ.get( False ) except Empty: break
class TCPClientManager(object): """A Client for the 'Push' feature in Device Cloud""" def __init__(self, conn, secure=True, ca_certs=None, workers=1): """ Arbitrator for multiple TCP Client Sessions :param conn: The :class:`devicecloud.DeviceCloudConnection` to use :param secure: Whether or not to create a secure SSL wrapped session. :param ca_certs: Path to a file containing Certificates. If not provided, the devicecloud.crt file provided with the module will be used. In most cases, the devicecloud.crt file should be acceptable. :param workers: Number of workers threads to process callback calls. """ self._conn = conn self._secure = secure self._ca_certs = ca_certs # A dict mapping Sockets to their PushSessions self.sessions = {} # IO thread is used monitor sockets and consume data. self._io_thread = None # Writer thread is used to send data on sockets. self._writer_thread = None # Write queue is used to queue up data to write to sockets. self._write_queue = Queue() # A pool that monitors callback events and invokes them. self._callback_pool = CallbackWorkerPool(self._write_queue, size=workers) self.closed = False self.log = logging.getLogger(__name__) @property def hostname(self): return self._conn.hostname @property def username(self): return self._conn.username @property def password(self): return self._conn.password def _restart_session(self, session): """Restarts and re-establishes session :param session: The session to restart """ # remove old session key, if socket is None, that means the # session was closed by user and there is no need to restart. if session.socket is not None: self.log.info("Attempting restart session for Monitor Id %s." % session.monitor_id) del self.sessions[session.socket.fileno()] session.stop() session.start() self.sessions[session.socket.fileno()] = session def _writer(self): """ Indefinitely checks the writer queue for data to write to socket. """ while not self.closed: try: sock, data = self._write_queue.get(timeout=0.1) self._write_queue.task_done() sock.send(data) except Empty: pass # nothing to write after timeout except socket.error as err: if err.errno == errno.EBADF: self._clean_dead_sessions() def _clean_dead_sessions(self): """ Traverses sessions to determine if any sockets were removed (indicates a stopped session). In these cases, remove the session. """ for sck in list(self.sessions.keys()): session = self.sessions[sck] if session.socket is None: del self.sessions[sck] def _select(self): """ While the client is not marked as closed, performs a socket select on all PushSession sockets. If any data is received, parses and forwards it on to the callback function. If the callback is successful, a PublishMessageReceived message is sent. """ try: while not self.closed: try: inputready = select.select(self.sessions.keys(), [], [], 0.1)[0] for sock in inputready: session = self.sessions[sock] sck = session.socket if sck is None: # Socket has since been deleted, continue continue # If no defined message length, nothing has been # consumed yet, parse the header. if session.message_length == 0: # Read header information before receiving rest of # message. response_type = _read_msg_header(session) if response_type == NO_DATA: # No data could be read, assume socket closed. if session.socket is not None: self.log.error( "Socket closed for Monitor %s." % session.monitor_id) self._restart_session(session) continue elif response_type == INCOMPLETE: # More Data to be read. Continue. continue elif response_type != PUBLISH_MESSAGE: self.log.warn( "Response Type (%x) does not match PublishMessage (%x)" % (response_type, PUBLISH_MESSAGE)) continue try: if not _read_msg(session): # Data not completely read, continue. continue except PushException as err: # If Socket is None, it was closed, # otherwise it was closed when it shouldn't # have been restart it. session.data = six.b("") session.message_length = 0 if session.socket is None: del self.sessions[sck] else: self.log.exception(err) self._restart_session(session) continue # We received full payload, # clear session data and parse it. data = session.data session.data = six.b("") session.message_length = 0 block_id = struct.unpack('!H', data[0:2])[0] compression = struct.unpack('!B', data[4:5])[0] payload = data[10:] if compression == 0x01: # Data is compressed, uncompress it. payload = zlib.decompress(payload) # Enqueue payload into a callback queue to be # invoked self._callback_pool.queue_callback( session, block_id, payload) except select.error as err: # Evaluate sessions if we get a bad file descriptor, if # socket is gone, delete the session. if err.args[0] == errno.EBADF: self._clean_dead_sessions() except Exception as err: self.log.exception(err) finally: for session in self.sessions.values(): if session is not None: session.stop() def _init_threads(self): """Initializes the IO and Writer threads""" if self._io_thread is None: self._io_thread = Thread(target=self._select) self._io_thread.start() if self._writer_thread is None: self._writer_thread = Thread(target=self._writer) self._writer_thread.start() def create_session(self, callback, monitor_id): """ Creates and Returns a PushSession instance based on the input monitor and callback. When data is received, callback will be invoked. If neither monitor or monitor_id are specified, throws an Exception. :param callback: Callback function to call when PublishMessage messages are received. Expects 1 argument which will contain the payload of the pushed message. Additionally, expects function to return True if callback was able to process the message, False or None otherwise. :param monitor_id: The id of the Monitor, will be queried to understand parameters of the monitor. """ self.log.info("Creating Session for Monitor %s." % monitor_id) session = SecurePushSession(callback, monitor_id, self, self._ca_certs) \ if self._secure else PushSession(callback, monitor_id, self) session.start() self.sessions[session.socket.fileno()] = session self._init_threads() return session def stop(self): """Stops all session activity. Blocks until io and writer thread dies """ if self._io_thread is not None: self.log.info("Waiting for I/O thread to stop...") self.closed = True self._io_thread.join() if self._writer_thread is not None: self.log.info("Waiting for Writer Thread to stop...") self.closed = True self._writer_thread.join() self.log.info("All worker threads stopped.")
class OvnDbNotifyHandler(object): STOP_EVENT = ("STOP", None, None, None) def __init__(self, driver): self.driver = driver self.__watched_events = set() self.__lock = threading.Lock() self.notifications = Queue() self.notify_thread = greenthread.spawn_n(self.notify_loop) atexit.register(self.shutdown) def matching_events(self, event, row, updates): with self.__lock: return tuple(t for t in self.__watched_events if t.matches(event, row, updates)) def watch_event(self, event): with self.__lock: self.__watched_events.add(event) def watch_events(self, events): with self.__lock: for event in events: self.__watched_events.add(event) def unwatch_event(self, event): with self.__lock: try: self.__watched_events.remove(event) except KeyError: # For ONETIME events, they should normally clear on their own pass def unwatch_events(self, events): with self.__lock: for event in events: try: self.__watched_events.remove(event) except KeyError: # For ONETIME events, they should normally clear on # their own pass def shutdown(self): self.notifications.put(OvnDbNotifyHandler.STOP_EVENT) def notify_loop(self): while True: try: match, event, row, updates = self.notifications.get() if (not isinstance(match, row_event.RowEvent) and (match, event, row, updates) == ( OvnDbNotifyHandler.STOP_EVENT)): self.notifications.task_done() break match.run(event, row, updates) if match.ONETIME: self.unwatch_event(match) self.notifications.task_done() except Exception: # If any unexpected exception happens we don't want the # notify_loop to exit. LOG.exception(_LE('Unexpected exception in notify_loop')) def notify(self, event, row, updates=None): matching = self.matching_events( event, row, updates) for match in matching: self.notifications.put((match, event, row, updates))
class BasicSpider: def __init__(self, max_task, url_page_format, max_page=1000, start_page=1): self.next_page = start_page self.start_page = start_page self.max_task = max_task self.max_page = max_page self._queue = Queue() self.url_page_format = url_page_format self.page_number_data = [None for _ in range(max_page)] def _task_controller(self): task_done_counter = 0 while True: page, _ = self._queue.get() self._queue.task_done() if page is not None: task_done_counter += 1 print(page, task_done_counter) if task_done_counter % 50 == 0: self.save_addup_data('./data/%s-%s.txt' % (__name__, time.time())) if task_done_counter + self.start_page - 1 >= self.max_page: self.save_addup_data('./data/%s-%s.txt' % (__name__, time.time())) print(self.get_total()) break else: if self.next_page <= self.max_page: print('NEXT_PAGE: %d' % self.next_page) self._run_next_page() def _spider_handler(self, page, **kwargs): while not self._addup_wrapper(page, **kwargs): pass self._queue.put_nowait((page, kwargs)) def _run_next_page(self, kwargs=None): if self.next_page > self.max_page: return while self.page_number_data[self.next_page - 1] is not None: self.next_page += 1 if self.next_page > self.max_page: self._queue.put_nowait((None, kwargs or {})) return threading.Thread(target=self._spider_handler, args=(self.next_page, ), kwargs=kwargs).start() self.next_page += 1 def run(self): for i in range(self.max_task): self._run_next_page() time.sleep(0.1) ctrl = threading.Thread(target=self._task_controller, name='task_controller') ctrl.start() # for inspection while True: time.sleep(1) def _addup_wrapper(self, page, **kwargs): try: self.get_page_addup(page, **kwargs) except Exception as e: print('===========error') print(page) print(e) # print_exc() print('===========error') else: return True return False def save_addup_data(self, name): tl = [str(i) for i in self.page_number_data] with open(name, 'w') as f: f.write('\n'.join(tl)) def get_bs4(self, page): req = requests.get(self.url_page_format % page, headers=HEADERS) text = req.text bs4 = BeautifulSoup(text, 'lxml') return bs4 def get_page_addup(self, page, **kwargs): return def get_total(self): if list(self.check_page_number()): raise ValueError('addup data is incomplete ') counter = 0 for i in self.page_number_data: counter += i return counter def check_page_number(self): for i, v in enumerate(self.page_number_data): if v is None: yield i def load_page_data(self, filepath): self.page_number_data = [] with open(filepath, 'r') as f: data_str = f.read() for i in data_str.split('\n'): if i: if i.strip() == 'None' or i.strip() == '0': self.page_number_data.append(None) else: self.page_number_data.append(int(i.strip()))