class StreamReader(Thread): MAX_SHUTDOWN_COUNTER = 6 POLL_TIMEOUT = 1.0 def __init__(self, server, topic, group_id): self._consumer = Consumer({ 'bootstrap.servers': server, 'group.id': group_id, 'auto.offset.reset': 'earliest' }) self._consumer.subscribe([topic]) self._shutdown_counter = 0 self._queue = SimpleQueue() super().__init__() def fetch(self, block=True, timeout=None): while True: msg = self._consumer.poll(StreamReader.POLL_TIMEOUT) if msg is None or msg.error(): self._shutdown_counter += 1 if self._shutdown_counter == StreamReader.MAX_SHUTDOWN_COUNTER: break else: yield msg.value().decode('utf-8') def run(self): for message in self.fetch(): self._queue.put_nowait(message) self._queue.put_nowait(None) def queue(self): return self._queue
def breadth_first_spelunking(player, graph): breadcrumbs = SimpleQueue() visited_rooms = set([]) breadcrumbs.put_nowait([player.current_room.id]) # retrace path as long as there are breadcrumbs in the queue while not breadcrumbs.qsize() is 0: path = breadcrumbs.get_nowait() last_room = path[-1] # Check if we hit an unexplored room if last_room not in visited_rooms: # Log it visited_rooms.add(last_room) # Check exits for room_exit in graph[last_room]: # Return path if we find an unexplored exit if graph[last_room][room_exit] is "?": return path # Otherwise, keep on truckin' else: breadcrumb_path = list(path) breadcrumb_path.append(graph[last_room][room_exit]) breadcrumbs.put_nowait(breadcrumb_path) # when we run out of breadcrumbs in our queue, return an empty list # as we exit the loop since we're back at an unexplored path return list()
def __init__(self, create_func: Callable, create_args: Tuple = None, create_kwargs: Dict = None, min_size: int = 1, max_size: int = 4, name: str = None) -> None: assert create_func, 'A create function must be provided' assert 1 <= min_size <= max_size <= 32, ( f'Pool size out of range: min={min_size}, max={max_size}') if create_args is None: create_args = () if create_kwargs is None: create_kwargs = {} create_func = functools.partial(create_func, *create_args, **create_kwargs) pool = PoolQueue() for conn in [create_func() for _ in range(min_size)]: pool.put_nowait(conn) self._create_func = create_func self._pool = pool self._size = min_size self._closed = False self._lock = threading.Lock() self.min_size = min_size self.max_size = max_size self.name = name
def processed_reader(func, items_to_read, support_files): # pragma: no cover process_pool = [] slots = 8 reply_queue = SimpleQueue() send_queue = SimpleQueue() for item_index in range(slots): if item_index < len(items_to_read): send_queue.put(items_to_read[item_index]) for i in range(slots): process = threading.Thread( target=_inner_process, args=(func, send_queue, reply_queue), ) process.daemon = True process.start() process_pool.append(process) process_start_time = time.time() item_index = slots while any({p.is_alive() for p in process_pool}): try: records = b"" while 1: records = reply_queue.get_nowait() if records == b"END OF RECORDS": break yield from map(json, records) if item_index < len(items_to_read): # we use this mechanism to throttle reading blobs so we # don't exhaust memory send_queue.put_nowait(items_to_read[item_index]) item_index += 1 else: send_queue.put_nowait(TERMINATE_SIGNAL) except Empty: # nosec if time.time( ) - process_start_time > MAXIMUM_SECONDS_PROCESSES_CAN_RUN: logging.error( f"Sending TERMINATE to long running multi-processed processes after {MAXIMUM_SECONDS_PROCESSES_CAN_RUN} seconds total run time" ) break except GeneratorExit: logging.error("GENERATOR EXIT DETECTED") break for process in process_pool: process.join()
class StatsReporter(threading.Thread): def __init__(self, report_interval: int): super().__init__() self.report_interval = report_interval self.stop = threading.Event() self.stats_queue = SimpleQueue() def run(self): while not self.stop.wait(self.report_interval): pool_batch_stats = defaultdict(list) while not self.stats_queue.empty(): pool_uid, batch_stats = self.stats_queue.get() pool_batch_stats[pool_uid].append(batch_stats) total_processed_batches = sum( len(pool_stats) for pool_stats in pool_batch_stats.values()) logger.info( f'Processed {total_processed_batches} batches in last {self.report_interval} seconds:' ) for pool_uid, pool_stats in pool_batch_stats.items(): total_batches = len(pool_stats) total_examples = sum(batch_stats.batch_size for batch_stats in pool_stats) avg_batch_size = mean(batch_stats.batch_size for batch_stats in pool_stats) total_time = sum(batch_stats.processing_time for batch_stats in pool_stats) batches_to_time = total_batches / total_time batch_performance = f'{batches_to_time:.2f} ' + ( 'batches/s' if batches_to_time > 1 else 's/batch') examples_to_time = total_examples / total_time example_performance = f'{examples_to_time:.2f} ' + ( 'examples/s' if examples_to_time > 1 else 's/example') logger.info( f'{pool_uid}: ' f'{total_batches} batches ({batch_performance}), ' f'{total_examples} examples ({example_performance}), ' f'avg batch size {avg_batch_size:.2f}') def report_stats(self, pool_uid, batch_size, processing_time): batch_stats = BatchStats(batch_size, processing_time) self.stats_queue.put_nowait((pool_uid, batch_stats))
class ThreadedFunction(Thread): def __init__(self, function: Callable) -> None: super().__init__(daemon=True) self.__stop_event = Event() self.__function = function self.__call_queue = Queue() self.__result_map: Dict[UUID, Result] = dict() def run(self) -> None: while not self.__stop_event.is_set(): uuid, args, kwargs = self.__call_queue.get() if (args != ()): if (kwargs != {}): self.__result_map[uuid].resolve( self.__function(*args, **kwargs)) else: self.__result_map[uuid].resolve(self.__function(*args)) else: if (kwargs != {}): self.__result_map[uuid].resolve(self.__function(**kwargs)) else: self.__result_map[uuid].resolve(self.__function()) def stop(self) -> None: self.__stop_event.set() Thread.join(self) def __call__(self, *args, **kwargs) -> Any: uuid = uuid4() result = Result() self.__call_queue.put_nowait((uuid, args, kwargs)) self.__result_map[uuid] = result return result def __del__(self) -> None: self.stop()
class Consumer: def __init__(self, broker, schema_registry, topic, logging_enabled=False, groupId="asgardConsumerGroup", autocommit=True): """ Initialiser for Confluent Consumer using AvroConsumer. Each consumer can only be subscribed to one topic Parameters ---------- broker: str The URL of the broker (example: 'localhost:9092') schema_registry: str The URL of the confluent Schema Registry endpoint (example: 'http://localhost:8081') topic: str The topic to subscribe too logger: Logger object, Optional The logger object which will be used to log messages if provided groupId: str, Optional An optional groupId which can be used to loadbalance consumers default is "asgard" """ """self.__consumer = AvroConsumer( { "bootstrap.servers": broker, "group.id": groupId, "schema.registry.url": schema_registry, "enable.auto.commit": autocommit } )""" self.__consumer = KafkaConsumer({ "bootstrap.servers": broker, "group.id": groupId, "enable.auto.commit": autocommit, "auto.offset.reset": "latest" }) self.autocommit = autocommit if not autocommit: self.consumed_messages = SimpleQueue() self.__consumer.subscribe([topic]) if logging_enabled: self.logger = logging.getLogger(__name__) else: self.logger = None def consume(self): """ Method to consume and return message if exists and can be deserialized Returns ------- str The recieved message payload as a string None No message has been recieved or an error has occured """ msg = None try: msg = self.__consumer.poll(1) except SerializerError as e: self.__log_msg("Message deserialization has failed {}: {}".format( msg, e), "See the following stack trace", f"{traceback.format_exc()}", delimeter="\n", level="ERROR") except RuntimeError as e: self.__log_msg( "The consumer has been closed and cannot recieve messages", level="ERROR") except Exception as e: self.__log_msg("An unkown error has occured {}".format(e), "See the following stack trace", f"{traceback.format_exc()}", delimeter="\n", level="ERROR") if not msg is None: if msg.error(): self.__log_msg("AvroConsumer error: {}".format(msg.error()), level="ERROR") else: if not self.autocommit: self.consumed_messages.put_nowait(msg) return json.loads(msg.value().decode()).get("payload") def __enter__(self): return self.__consumer def __exit__(self, *args): self.close() def __log_msg( self, *messages, level="NOTSET", delimeter=" ", ): levels = { "CRITICAL": logging.CRITICAL, "ERROR": logging.ERROR, "WARNING": logging.WARNING, "INFO": logging.INFO, "DEBUG": logging.DEBUG, "NOTSET": logging.NOTSET } msg = delimeter.join(messages) if self.logger is not None: if level not in levels: raise ValueError( f"level {level} is not valid must be one of {list(levels.keys())}" ) self.logger.log(levels[level], msg) else: if level is not None: print(f"LOGGED MESSAGE: {msg}") else: print(f"{level}: {msg}") def commit(self, asynchronous=True): if not self.autocommit and not self.consumed_messages.empty(): msg = self.consumed_messages.get_nowait() self.__consumer.commit(msg, asynchronous=asynchronous) def close(self): """ Close the consumer, Once called this object cannot be reused """ self.__consumer.close()
class ThreadPoolExecutor(ThreadPoolExecutorBase): __slots__ = ( "__futures", "__pool", "__tasks", "__write_lock", "__thread_events", ) def __init__( self, max_workers: int = max((cpu_count(), 4)), loop: AbstractEventLoop = None, ) -> None: """""" if loop: warnings.warn(DeprecationWarning("loop argument is obsolete")) self.__futures = set() # type: typing.Set[Future[Any]] self.__thread_events = set() # type: typing.Set[threading.Event] self.__tasks = SimpleQueue() # type: SimpleQueue[Optional[WorkItem]] self.__write_lock = threading.RLock() pools = set() for idx in range(max_workers): pools.add(self._start_thread(idx)) self.__pool = frozenset(pools, ) # type: typing.FrozenSet[threading.Thread] def _start_thread(self, idx: int) -> threading.Thread: event = threading.Event() self.__thread_events.add(event) thread = threading.Thread( target=self._in_thread, name="[%d] Thread Pool" % idx, args=(event, ), ) thread.daemon = True thread.start() return thread def _in_thread(self, event: threading.Event) -> None: while True: work_item = self.__tasks.get() if work_item is None: break try: if work_item.loop.is_closed(): log.warning( "Event loop is closed. Call %r skipped", work_item.func, ) continue work_item() except asyncio.CancelledError: break finally: del work_item event.set() def submit( # type: ignore self, fn: F, *args: Any, **kwargs: Any) -> Future: """ Submit blocking function to the pool """ if fn is None or not callable(fn): raise ValueError("First argument must be callable") loop = asyncio.get_event_loop() with self.__write_lock: future = loop.create_future() # type: asyncio.Future[Any] self.__futures.add(future) future.add_done_callback(self.__futures.remove) self.__tasks.put_nowait( WorkItem( func=fn, args=args, kwargs=kwargs, future=future, loop=loop, ), ) return future def shutdown(self, wait: bool = True) -> None: for _ in self.__pool: self.__tasks.put_nowait(None) for f in filter(lambda x: not x.done(), self.__futures): f.set_exception(ThreadPoolException("Pool closed")) if not wait: return while not all(e.is_set() for e in self.__thread_events): time.sleep(0) def _adjust_thread_count(self) -> None: raise NotImplementedError def __del__(self) -> None: self.shutdown()
class ThreadPoolExecutor(ThreadPoolExecutorBase): __slots__ = ( '__futures', '__pool', '__tasks', '__write_lock', '__thread_events', ) def __init__(self, max_workers=max((cpu_count(), 4)), loop=None): if loop: warnings.warn(DeprecationWarning("loop argument is obsolete")) self.__futures = set() self.__pool = set() self.__thread_events = set() self.__tasks = SimpleQueue() self.__write_lock = threading.RLock() for idx in range(max_workers): self.__pool.add(self._start_thread(idx)) self.__pool = frozenset(self.__pool) def _start_thread(self, idx): event = threading.Event() self.__thread_events.add(event) thread = threading.Thread(target=self._in_thread, name="[%d] Thread Pool" % idx, args=(event, )) thread.daemon = True thread.start() return thread def _in_thread(self, event: threading.Event): while True: work_item = self.__tasks.get() if work_item is None: break try: if work_item.loop.is_closed(): log.warning("Event loop is closed. Call %r skipped", work_item.func) continue work_item() except asyncio.CancelledError: break finally: del work_item event.set() def submit(self, fn, *args, **kwargs): if fn is None or not callable(fn): raise ValueError('First argument must be callable') loop = asyncio.get_event_loop() with self.__write_lock: future = loop.create_future() # type: asyncio.Future self.__futures.add(future) future.add_done_callback(self.__futures.remove) self.__tasks.put_nowait( WorkItem(func=fn, args=args, kwargs=kwargs, future=future, loop=loop)) return future def shutdown(self, wait=True): for _ in self.__pool: self.__tasks.put_nowait(None) for f in filter(lambda x: not x.done(), self.__futures): f.set_exception(ThreadPoolException("Pool closed")) if not wait: return while not all(e.is_set() for e in self.__thread_events): time.sleep(0) def _adjust_thread_count(self): raise NotImplementedError def __del__(self): self.shutdown()
class Connection: """Describes a connection either from the server to some client or from the client to the server Attributes: connection (socket.socket): how we communicate with the other entity address (str): where the entity connected from / where we connected to send_queue (queue[bytes]): the packets that we need to send rec_queue (queue[Packet]): the packets that they have sent us curr_send_packet (optional BytesIO): if we are currently trying to send a message to the client, this is the serialized message we are trying to send (that has already been removed from the send_queue) curr_rec (deque[bytes]): the things that we have in memory received """ def __init__(self, connection: socket.socket, address: str) -> None: self.connection = connection self.address = address self.send_queue = Queue() self.rec_queue = Queue() self.curr_send_packet: io.BytesIO = None self.curr_rec = deque() def disconnected(self): """Returns True if the connection is dead for whatever reason, False otherwise""" return self.connection is None def update(self): """Handles sending and receiving packets in a non-blocking way. Must be called very regularly for send() and receive() to actually do anything """ if self.disconnected(): return try: self._handle_send() self._handle_rec() except BlockingIOError: pass except OSError: self.connection = None print(f'[networking.shared] connection lost') traceback.print_exc() def _handle_send(self): if self.curr_send_packet is None: if self.send_queue.empty(): return packet_serd = self.send_queue.get_nowait() self.curr_send_packet = io.BytesIO() self.curr_send_packet.write( len(packet_serd).to_bytes(4, 'big', signed=False)) self.curr_send_packet.write(packet_serd) self.curr_send_packet.seek(0, 0) for _ in range(128): # avoid sending more than 512kb in one go block = self.curr_send_packet.read(BLOCK_SIZE) if not block: self.curr_send_packet = None return amt_sent = self.connection.send(block) if amt_sent < len(block): self.curr_send_packet.seek(amt_sent - len(block), 1) return def _try_from_recq(self, amt: int) -> typing.Optional[bytes]: """Tries to read the specified number of bytes from the receive queue. If this fails to get that many bytes the receive queue is effectively unaltered, otherwise the bytes are removed from the receive queue and returned""" if not self.curr_rec: return None if len(self.curr_rec) == 1 or len(self.curr_rec[0]) >= amt: # happy / most common case if len(self.curr_rec[0]) < amt: return None block = self.curr_rec.popleft() if len(block) == amt: return block self.curr_rec.appendleft(block[amt:]) return block[:amt] result = io.BytesIO() curlen = 0 while self.curr_rec: block = self.curr_rec.popleft() if curlen + len(block) == amt: # another happy / common case result.write(block) return result.getvalue() if curlen + len(block) < amt: result.write(block) continue result.write(block[:amt]) self.curr_rec.appendleft(block[amt:]) return result.getvalue() # didn't get enough data, but now the curr_rec queue is all merged # so we will get the top happy case self.curr_rec.appendleft(result.getvalue()) return None def _handle_rec(self): for _ in range(128): # avoid reading more than 512kb in one go block = self.connection.recv(BLOCK_SIZE) if not block: self.connection.close() self.connection = None break self.curr_rec.append(block) if len(block) < BLOCK_SIZE: break for _ in range(8): # avoid parsing too many packets at once lenblock = self._try_from_recq(4) if not lenblock: return explen = int.from_bytes(lenblock, 'big', signed=False) block = self._try_from_recq(explen) if not block: self.curr_rec.appendleft(lenblock) return packet = ser.deserialize(block) if not isinstance(packet, packets.Packet): raise ValueError( f'got non-packet {packet} (type={type(packet)})') self.rec_queue.put(packet) def send(self, packet: packets.Packet): """Sends this client the specified packet""" if self.disconnected(): return self.send_queue.put_nowait(ser.serialize(packet)) def send_serd(self, packet_serd: bytes): """Sends this client the serialized packet""" if self.disconnected(): return self.send_queue.put_nowait(packet_serd) def read(self) -> typing.Optional[packets.Packet]: """Returns the packet from the client if there is one""" return self.rec_queue.get_nowait( ) if not self.rec_queue.empty() else None def has_pending(self, read=True, write=True) -> bool: """Returns True if there are pending sends / receives, False otherwise""" if write and not self.send_queue.empty(): # have things to send still return True if read and not self.rec_queue.empty(): # have things that we've parsed but haven't been read() yet return True if write and self.curr_send_packet is not None: # in the middle of sending something return True if read and self.curr_rec: # have things not yet parsed / incomplete return True return False
class Watcher(): @staticmethod def __openSerialPort__(port: str, verbose=False) -> SerialPort: serialPort = SerialPort(**options) serialPort.port = port try: serialPort.open() if verbose: print("Port {0} is opened".format(port)) except Exception as e: print('Problem opening port ' + port) print(e) exit(-1) return serialPort @property def verbose(self) -> bool: return self.__startupArguments__.verbose @property def kind(self) -> str: return str(self.__startupArguments__.kind) def log(self, msg): console_log("{}:{}".format(self.kind, msg)) def __init__(self, args: StartupArguments): self.__startupArguments__ = args self.__msgQueue__ = Queue() self.__input__ = self.__openSerialPort__(args.portFrom, args.verbose) self.__output__ = self.__openSerialPort__(args.portTo, args.verbose) self.__buffer__ = b'' self.__sequenceEnding__ = False self.__parser__ = ParserFactory.createParser(args.kind) self.__sio__ = socketio.AsyncClient(reconnection=True) self.__nsp__ = '/' + self.kind self.__sio__.register_namespace(SocketHubNsp(self.__nsp__)) self.__halt__ = False def __enqueueMessage__(self): self.__msgQueue__.put_nowait(self.__buffer__) async def __queueWatchCoro__(self) -> dict: message = "No data" buffer = b'' try: buffer = self.__msgQueue__.get(timeout=5) message = self.__parser__.parse(buffer) except EmptyQueueException: self.log('EmptyQueueException Raised') finally: msg_type = int(self.__parser__.get_type(message).value) data = {'status': message, 'type': msg_type} if 'Unknown sequence' in message: data['buffer'] = str(buffer) return data async def __queueWatchTask__(self): while 1: task = asyncio.ensure_future(self.__queueWatchCoro__()) data: dict = await task if self.verbose: self.log(data) if self.__sio__.connected: try: await self.__sio__.emit('event', data=data, namespace=self.__nsp__) except Exception as e: console_log('Emit error: ' + str(e)) self.__halt__ = True else: console_log('No connection to hub.') self.__halt__ = True async def __initializeSocketIOConnection__(self): try: await self.__sio__.connect('http://localhost:3000/', transports=['polling']) except Exception as e: console_log('Connection to hub failed: ' + str(e)) self.__halt__ = True def __queueWatchWorker__(self): loop = asyncio.new_event_loop() asyncio.ensure_future(self.__queueWatchTask__(), loop=loop) self.log('Queue Watch Task is running') loop.run_forever() def __socketIOWorker__(self): loop = asyncio.new_event_loop() asyncio.ensure_future(self.__initializeSocketIOConnection__(), loop=loop) loop.run_forever() # noinspection PyUnreachableCode def mainloop(self): self.log('Initializing...') queueWatcherThread = threading.Thread(target=self.__queueWatchWorker__) queueWatcherThread.setDaemon(True) queueWatcherThread.start() socketIOThread = threading.Thread(target=self.__socketIOWorker__) socketIOThread.setDaemon(True) socketIOThread.start() # Await for connection in sync manner while not self.__sio__.connected and not self.__halt__: pass if self.__sio__.connected: self.log('Watcher initialized!') else: self.log("Failed. Rerunning...") sys.exit(-1) try: while not self.__halt__: received_byte = self.__input__.read(size=1) if received_byte == b'\x02': self.__buffer__ = b'' elif received_byte == b'\x03': if self.__sequenceEnding__ and received_byte == b'\x03': self.__enqueueMessage__() self.__sequenceEnding__ = False elif self.__sequenceEnding__ and not received_byte == b'\x03' and self.verbose: self.log('Malformed message received') else: #if __sequenceEnding__ was not yet raised self.__sequenceEnding__ = True else: self.__buffer__ += received_byte self.__output__.write(received_byte) self.__output__.flush() except KeyboardInterrupt: self.log('Terminating...') self.log('Halt') sys.exit(-2) queueWatcherThread.join() socketIOThread.join()
def exec(self): logging.info('Start to find requests') visited = set() session = HTMLSession() queue = SimpleQueue() queue.put_nowait(self.args['url']) visited.add(self.args['url']) count = 0 while not queue.empty(): count += 1 if count > self.args['max_page_count']: break url = queue.get_nowait() logging.info('Request on {}'.format(url)) r = session.get(url, cookies=self.cookies) links = r.html.absolute_links for link in links: def is_exclude(): for exc in self.args['exclude']: if link.startswith(exc): return True return False if link.startswith(self.args['url'])\ and not is_exclude()\ and link not in visited: visited.add(link) queue.put_nowait(link) forms = r.html.find('form') for form in forms: request = { 'uuid': randuuid(), 'location': url, 'url': urljoin(r.url, form.attrs['action']) if 'action' in form.attrs else url, 'method': form.attrs['method'].upper() if 'method' in form.attrs else 'GET', 'content-type': form.attrs['enctype'] if 'enctype' in form.attrs else 'application/x-www-form-urlencoded', 'fields': {}, 'form': form } inputs = form.find('input') for inp in inputs: if 'name' not in inp.attrs: continue typ = inp.attrs['type'] if 'type' in inp.attrs else 'text' name = inp.attrs['name'] value = inp.attrs['value'] if 'value' in inp.attrs else '' required = True if 'required' in inp.attrs else False if typ == 'radio': if name in request['fields']: request['fields'][name]['values'].append(value) else: request['fields'][name] = { 'type': 'radio', 'name': name, 'required': required, 'values': [value] } elif typ == 'checkbox': checked = True if 'checked' in inp.attrs and ( inp.attrs['checked'] == 'checked' or inp.attrs['checked'] == '') else False request['fields'][name] = { 'type': 'checkbox', 'name': name, 'required': required, 'value': value, 'checked': checked } else: request['fields'][name] = { 'type': typ, 'name': name, 'required': required, 'default': value } textareas = form.find('textarea') for textarea in textareas: if 'name' not in textarea.attrs: continue name = textarea.attrs['name'] request['fields'][name] = { 'type': 'textarea', 'name': name, 'required': True if 'required' in textarea.attrs else False, 'default': textarea.text } selects = form.find('select') for select in selects: if 'name' not in select.attrs: continue name = select.attrs['name'] multiple = True if 'multiple' in select.attrs else False required = True if 'required' in select.attrs else False values = [] options = select.find('option') for option in options: if 'value' not in option.attrs: continue values.append(option.attrs['value']) request['fields'][name] = { 'type': 'select', 'name': name, 'multiple': multiple, 'required': required, 'values': values } logging.info('Found form {}'.format(request['uuid'])) self.requests[request['uuid']] = request self.results['requests'] = self.requests self.results['urls'] = list(visited) logging.info('Found {} forms'.format(len(self.results['requests'])))
class Producer: def __init__(self, *topics, broker, schema_registry, schema, logging_enabled = False): """ Initialization of the Producer which instatiates an AvroProducer class Parameters ---------- broker: str The URL of the broker (example: 'localhost:9092') schema_registry: str The URL of the confluent Schema Registry endpoint (example: 'http://localhost:8081') schema: str The default AVRO schema to use to serialize messages logger: Logger object, optional The logger object which will be used to log messages if provided topics variable length argument list of the string names of topics to produce too """ self.schema = avro.loads(schema) self.__producer = AvroProducer( { "bootstrap.servers": broker, "schema.registry.url": schema_registry }, default_key_schema=self.schema ) if logging_enabled: self.logger = logging.getLogger(__name__) else: self.logger = None self.topics = topics self.produce_flag = True self.production_last_stoped = 0 self.total_time_producing_stoped = 0 self.__msg_queue = SimpleQueue() def produce(self, msg, schema = None, callback = None): """ Write a message to confluent kafka using the instatiated AvroProducer to serialize Parameters ---------- msg: str The message to be serialized and sent schema: str, Optional An optional schema to overide the default set in the constructor callback: Function object, Optional An optional callback which will be executed whether the producing of the message fails or succeeds. This function must take two parameters the first for the error and the second for the message (https://docs.confluent.io/current/clients/confluent-kafka-python/#producer) """ # TODO: function partials for better readability ? # SOLVED: created dictionary to expand to parameters params = {} params["value"] = msg if schema is not None: params["value_schema"] = schema else: params["value_schema"] = self.schema if callback is not None: params["on_delivery"] = callback for topic in self.topics: params["topic"] = topic self.__msg_queue.put_nowait(params) try: while not self.__msg_queue.empty(): msg = self.__msg_queue.get_nowait() self.__producer.produce(**msg) self.__producer.flush() self.produce_flag = True self.production_last_stoped = 0 except SerializerError as e: self.__log_msg( "ERROR", "Message deserialization has failed {}: {} \n".format(msg,e), "See the following trace back \n {}".format(traceback.format_exc()) ) except BufferError as e: if self.produce_flag or ( self.production_last_stoped != 0 and ((time.time() - self.production_last_stoped) >= 3600) ): self.produce_flag = False self.production_last_stoped = time.time() self.total_time_producing_stoped += time.time() - self.production_last_stoped self.__log_msg( "Queue Buffer has reached its maximum capacity, unable to deliver message {}: {}".format(msg,e), "Message production will be shut down until messages can be resent", f"Total time message producing has stopped {self.total_time_producing_stoped}", level="CRITICAL", delimeter="\n" ) except KafkaException as e: self.__log_msg( "An unknown exception has occured specific to Kafka {}: {}".format(msg, e), level="ERROR", delimeter="" ) except Exception as e: self.__log_msg( "An unknown exception has occured {}: {}".format(msg, e), f"See the following traceback {traceback.format_exc()}", level="ERROR", delimeter="\n" ) def __enter__(self): """ Context Manager for Producer, to allow custom actions for producing messages """ return self.__producer def __exit__(self, *args): """ On exit producer is flushed """ self.__producer.flush() def __log_msg(self, *messages, level="NOTSET", delimeter= " "): levels = { "CRITICAL": logging.CRITICAL, "ERROR": logging.ERROR, "WARNING": logging.WARNING, "INFO": logging.INFO, "DEBUG": logging.DEBUG, "NOTSET": logging.NOTSET } msg = delimeter.join(messages) if self.logger is not None: if level not in levels: raise ValueError( f"level {level} is not valid must be one of {list(levels.keys())}" ) self.logger.log( levels[level], msg ) else: if level is not None: print(f"LOGGED MESSAGE: {msg}") else: print(f"{level}: {msg}")
class Port: def __init__(self, parser, portNotOpenException, PortExceptionType = None): self.__PortExceptionType = PortExceptionType or type(portNotOpenException) self.__autoOpenOnWrite = False self.__connectionListeners = set() self.__debugRead = False self.__debugWrite = False self.__errorProcessor = print self.__packet = None self.__parser = parser self.__path = None self.__portNotOpenException = portNotOpenException self.__queue = SimpleQueue() self.__throw = False @property def autoOpenOnWrite(self): return self.__autoOpenOnWrite @autoOpenOnWrite.setter def autoOpenOnWrite(self, autoOpenOnWrite): self.__autoOpenOnWrite = autoOpenOnWrite @property def debugRead(self): return self.__debugRead @debugRead.setter def debugRead(self, debugRead): self.__debugRead = debugRead @property def debugWrite(self): return self.__debugWrite @debugWrite.setter def debugWrite(self, debugWrite): self.__debugWrite = debugWrite @property def errorProcessor(self): return self.__errorProcessor @errorProcessor.setter def errorProcessor(self, errorProcessor): self.__errorProcessor = errorProcessor @property def parser(self): return self.__parser @property def path(self): return self.__path @path.setter def path(self, path): self.__path = path @property def throw(self): return self.__throw @throw.setter def throw(self, throw): self.__throw = throw def Packet(self, **kw): return Packet(self.__parser.format, **kw) def addConnectionListener(self, connectionListener): self.__addRemoveConnectionListener(True, connectionListener) def addQueueItem(self, item): self.__queue.put_nowait(item) def close(self): if self.isOpen(): self._close() def isOpen(self): StaticUtils.notImplemented() def open(self, path = None, **kw): try: self._open(path, **kw) return True except self.__PortExceptionType as e: self._processError(e) finally: self.__throw = False def packet(self, **kw): self.__packet = self.Packet(**kw) return self def removeConnectionListener(self, connectionListener): self.__addRemoveConnectionListener(False, connectionListener) def processQueue(self): try: item = self.__queue.get_nowait() if isinstance(item, ConnectionEstablished): for connectionListener in self.__connectionListeners: connectionListener.connectionEstablished(self) elif isinstance(item, ConnectionLost): for connectionListener in self.__connectionListeners: connectionListener.connectionLost(self, item.e) elif isinstance(item, DataReceived): self.__parser.parse(item.data) elif isinstance(item, ProcessError): self.__errorProcessor(item.e) else: raise UnknownItem(item) except Empty: pass def write(self, packet = None, throw = None): if not packet: packet = self.__packet if throw is not None: self.__throw = throw try: if self.__debugWrite: print(packet) else: if not self.isOpen(): if self.__autoOpenOnWrite: self._open() else: raise self.__portNotOpenException self._write(packet) return True except self.__PortExceptionType as e: self._processError(e) finally: self.__packet = None self.__throw = False def _close(self): StaticUtils.notImplemented() def _open(self, path = None, **kw): StaticUtils.notImplemented() def _processError(self, e): if self.__throw: raise e if self.__errorProcessor: self.addQueueItem(ProcessError(e)) def _write(self, packet): StaticUtils.notImplemented() def __addRemoveConnectionListener(self, add, connectionListener): StaticUtils.assertInheritance(connectionListener, ConnectionListener, "connectionListener") getattr(self.__connectionListeners, "add" if add else "remove")(connectionListener)
class C2Websocket: def __init__(self, url, user_name, connection_type, master_password=None): self.url = url self.user_name = user_name self.connection_type = connection_type self.master_password = master_password self.recv_queues = {} self.send_queue = SimpleQueue() self.connection_string = self.connect_to_websocket() def get_url_connection(self): return '{}?name={}&masterPassword={}&userType={}'.format( self.url, self.user_name, self.master_password, self.connection_type) def connect_to_websocket(self): ws = websocket.create_connection(self.get_url_connection()) ws.settimeout(0.05) websocket_thread = threading.Thread( target=self._listen_websocket_messages, args=(ws, )) websocket_thread.start() return self.get_url_connection() def send_action(self, action, payload=None, drop_queued_action_responses=False, action_options=None): if drop_queued_action_responses: self.drop_queued_action_responses(action) action_to_send = {KEY_ACTION: action, KEY_PAYLOAD: payload} if action_options: for action_key, action_value in action_options.items(): action_to_send[action_key] = action_value self.send_queue.put_nowait(json.dumps(action_to_send)) def send_service(self, client_id, service, payload, drop_queued_service_responses=False): if drop_queued_service_responses: self.drop_queued_service_responses(ACTION_SEND_MESSAGE, client_id, service) payload[KEY_SERVICE] = service self.send_action(ACTION_SEND_MESSAGE, payload, action_options={KEY_SEND_MESSAGE_TO: client_id}) def recv_action_response(self, action, block=True, timeout=None): try: return self.recv_queues[action].get(block=block, timeout=timeout) except KeyError: if block: self.recv_queues[action] = SimpleQueue() self.recv_action_response(action, block) except KeyboardInterrupt: return def recv_service_response(self, client_id, service, block=True, timeout=None): try: return self.recv_queues[(ACTION_SEND_MESSAGE, client_id, service)].get(block=block, timeout=timeout) except KeyError: if block: self.recv_queues[(ACTION_SEND_MESSAGE, client_id, service)] = SimpleQueue() return self.recv_service_response(client_id, service, block) except KeyboardInterrupt: return def drop_queued_action_responses(self, action): self.recv_queues[action] = SimpleQueue() def drop_queued_service_responses(self, action, client_id, service): self.recv_queues[(action, client_id, service)] = SimpleQueue() def _add_service_message_to_recv_queue(self, action, client_id, service, payload): if (action, client_id, service) not in self.recv_queues: self.recv_queues[(action, client_id, service)] = SimpleQueue() self.recv_queues[(action, client_id, service)].put(payload) def _add_action_message_to_recv_queue(self, action, payload): if action not in self.recv_queues: self.recv_queues[action] = SimpleQueue() self.recv_queues[action].put(payload) def _listen_websocket_messages(self, ws): self.keep_listening = True try: while self.keep_listening: try: message = ws.recv() # print('Recv {} -> {}'.format(self.user_name, message)) except websocket._exceptions.WebSocketTimeoutException: if not self.send_queue.empty(): to_send = self.send_queue.get() ws.send(to_send) # print('Sent {} -> {}'.format(self.user_name, to_send)) continue try: json_message = json.loads(message) except json.decoder.JSONDecodeError as e: continue if KEY_ACTION not in json_message: continue action = json_message[KEY_ACTION] if KEY_PAYLOAD in json_message and KEY_FROM in json_message and KEY_SERVICE in json_message[ KEY_PAYLOAD]: self._add_service_message_to_recv_queue( action, json_message[KEY_FROM], json_message[KEY_PAYLOAD][KEY_SERVICE], json_message[KEY_PAYLOAD]) else: self._add_action_message_to_recv_queue( action, json_message[KEY_PAYLOAD]) time.sleep(1) ws.close() except websocket._exceptions.WebSocketConnectionClosedException: self.connect_to_websocket() def stop(self): self.keep_listening = False