def __init__(self, args): """ Create instance of application controller. :param args: raw arguments dictionary :param logger: logger function to use """ # parse queue configuration based on additional arguments passed conf = QueueConf() conf.setAllConf(args) # log options processed all_options = [ " %s -> %s" % (key, value) for key, value in conf.copy().items() ] logger.debug("Configuration:\n%s", "\n".join(all_options)) # resolve options either directly or through session and other services self.mongodb_url = self._validate_mongodb_url( conf.getConfString(const.OPT_MONGODB_URL)) self.working_dir = self._validate_working_dir( conf.getConfString(const.OPT_WORKING_DIR)) self.service_dir = self._validate_service_dir( conf.getConfString(const.OPT_SERVICE_DIR)) self.session = self._create_session(conf) self.client = pymongo.MongoClient(self.mongodb_url) self.api = RestApiDispatcher(self.session)
def init_tcp(self): self._tp = paramiko.Transport(self._socket, gss_kex=True) self._tp.set_gss_host(socket.getfqdn("")) self._tp.load_server_moduli() host_key = paramiko.RSAKey(filename=conf.SSH_SERVER_RSA_EKY) self._tp.add_server_key(host_key) server = Server() logger.debug('Create Server Class') try: self._tp.start_server(server=server) except paramiko.SSHException: logger.error("SSH negotiation failed.") gvar.manager.close_connection(self) exit() self._channel = self._tp.accept(200) if self._channel is None: logger.error('No channel') gvar.manager.close_connection(self) exit() logger.info('Authenticatied!')
def iteration(self): """ Run single iteration, entire logic of executor should be specified in this method, unless there is an additional logic between iterations. Iteration is cancelled, if executor is terminated. :return: boolean flag, True - run next iteration, False - terminate """ # we process special case of terminated executor in case someone would launch it again. if self._terminated: logger.warning("Executor %s has been terminated", self.name) return False logger.debug("%s - Run iteration, timeout=%s", self.name, self.timeout) try: # send reponse to the scheduler that this executor is up and processing tasks self._respond_is_alive() # check if there are any messages in connection, process one message per iteration if self.conn.poll(): self._process_message(self.conn.recv()) # check if there is any outstanding task to run, otherwise poll data for current task self._process_task() except ExecutorInterruptedException: logger.info("%s - Requested termination of executor", self.name) self._terminated = True # cancel task that is currently running and clean up state self._cancel_active_task() return False # pylint: disable=W0703,broad-except except Exception as e: logger.exception("%s - Unrecoverable error %s, terminating", self.name, e) self._terminated = True return False # pylint: enable=W0703,broad-except else: return True
def run(self): for i in range(0, self._iterations): if self.is_cancelled(): logger.debug("Cancelled on iteration %s", i) break logger.debug("Iteration %s", i) time.sleep(1.0)
def parse_or_expression(self) -> Result: """ or_exp := or_exp OR and_exp """ r1 = self.parse_and_expression() if not r1.name and not r1.value: return r1 while True: token = self.pop_token() if token.type == TOKEN_TYPE['eof']: break elif token.type != TOKEN_TYPE['or']: self.push_token(token) return r1 r2 = self.parse_and_expression() if not r2.name and not r2.value: raise ParseException('invalid condition "%s"', self.condstr) r1 = Result('({} or {})'.format( r1.name, r2.name), r1.value or r2.value) logger.debug('[*] {}'.format(r1)) return r1
def recv(self): try: stream = self._socket.recv(1024) except OSError: logger.debug('telnet connect socket was closed.') raise SocketClosed return stream
def send_request(self): bs = self.get_proto_head(0x0) self.send(bs) self.status = self.WAIT_ROOM_ID_STATUS logger.debug('Regist: send request, bs:%s ,now status %s' % (bs.get_packet(), self.status))
def measure_point(self) -> Optional[Tuple[float, float]]: self.received = 0 rssi_values: List[int] = [] start = time.time() while time.time() - start < self.timeout and self.received < self.points_number: bytes_to_read = self.serial_conn.ser.inWaiting() text_read = self.serial_conn.ser.read(bytes_to_read).decode( encoding="utf-8" ) if text_read: args = text_read.split(" ") for arg in args: if "RSSI" in arg: rssi = arg.split("=")[1].strip() logger.debug(f"Read RSSI value from serial: {rssi}") try: rssi_values.append(int(rssi)) self.received += 1 except Exception as ex: print(ex) time.sleep(0.1) try: return ( statistics.mean(rssi_values), (len(rssi_values) / self.points_number) * 100, ) except statistics.StatisticsError: return None
def start_listening(self): self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.listener.bind(('127.0.0.1', self._listen_port)) self.listener.listen() logger.debug('Telnet.Start_linsten OK...')
def write(self, c: int): c = chr(c).encode() try: self.port.write(c) # except serial.serialutil.SerialTimeoutException: except Exception: logger.debug("Serial write timeout") self.close()
def recv(self): res = self.room.recv() logger.debug('Register recv message: %s' % res) p = Protocol(res) first_code = p.get_int8() if first_code != 0x1: return b'' # ! bad idea return p
def test(self, components: Tuple[str]): self.load_plugins() for component in self.iter_components(): if not component.name in components: continue logger.debug("test '%s' check matches", component.name) self._process_check_matches_result(component) self._process_implies() return self.results
def recv_from_control(self, msg): """ recv from control, then send to channel """ try: self.send_to_channel(msg) except OSError: logger.debug("send error, close this room: %s" % self.id) gvar.mansion.del_room_by_id(self.id)
def send_room_id(self): """ | 0x1 | room_id | server terminal listening port | """ bs = self.get_protocol_head(0x1) bs.add_str(self._room.id) bs.add_str(str(conf.SSH_SERVER_LISTENING_TERMINIAL_PORT)) self.send(bs) self.status = self.WAIT_ROOM_ID_STATUS logger.debug('Register: send room id: %s' % bs.get_packet())
def run(self): """ alias self.recv() """ try: self.recv() except Exception: logger.debug("connection got a error, closed") gvar.manager.close_connection(self) exit()
def update_map_with_rssi_values( self, id: int, rssi: float, perc: float ) -> shp.ShapeRecord: logger.debug(f"Update id {id} with RSSI {rssi} and {perc}%") for shaperec in self.shape_records: if shaperec.record["ID"] == id: shaperec.record["RSSI"] = rssi shaperec.record["PERC"] = perc return shaperec raise IndexError(f"Given id ({id}) was not found")
def get_fin_room_id(self, pro: Protocol): if not self.status == self.WAIT_ROOM_ID_STATUS: self.send_error() return room_id = pro.get_str() if not self._room.id == room_id: self.send_error() return self.status = self.FULL_STATUS logger.debug('Register: Full Status')
def interpret_group_8(self, instruction: int) -> None: # Logic and arithmetic operations between Vx and Vy x = (instruction & 0x0F00) >> 8 y = (instruction & 0x00F0) >> 4 last_nibble = instruction & 0x000F logger.debug( f"instruction:{instruction:X} x:{x:X}(0x{self.reg_v[x]:X})" f" y:{y:X}(0x{self.reg_v[y]:X})") if last_nibble == 0x0: # LD Vx, Vy self.reg_v[x] = self.reg_v[y] elif last_nibble == 0x1: # OR Vx, Vy self.reg_v[x] = self.reg_v[x] | self.reg_v[y] elif last_nibble == 0x2: # AND Vx, Vy self.reg_v[x] = self.reg_v[x] & self.reg_v[y] elif last_nibble == 0x3: # XOR Vx, Vy self.reg_v[x] = self.reg_v[x] ^ self.reg_v[y] elif last_nibble == 0x4: # ADD Vx, Vy # Vf is used as carry if self.reg_v[x] + self.reg_v[y] > 0xFF: self.reg_v[0xF] = 1 else: self.reg_v[0xF] = 0 self.reg_v[x] = (self.reg_v[x] + self.reg_v[y]) % 0x100 elif last_nibble == 0x5: # SUB Vx, Vy if self.reg_v[x] <= self.reg_v[y]: tmp = self.reg_v[x] - self.reg_v[y] self.reg_v[x] = 0x100 + tmp # + since tmp is negative self.reg_v[0xF] = 0 else: self.reg_v[x] = self.reg_v[x] - self.reg_v[y] self.reg_v[0xF] = 1 elif last_nibble == 0x6: # SHR Vx {, Vy} # Vf shall be set to same value as the bit that is shifted out if self.shift_quirks: self.reg_v[x] = self.reg_v[y] self.reg_v[0xF] = self.reg_v[x] & 0x0001 self.reg_v[x] = self.reg_v[x] >> 1 elif last_nibble == 0x7: # SUBN Vx, Vy if self.reg_v[y] <= self.reg_v[x]: tmp = self.reg_v[y] - self.reg_v[x] self.reg_v[x] = 0x100 + tmp self.reg_v[0xF] = 0 else: self.reg_v[x] = self.reg_v[y] - self.reg_v[x] self.reg_v[0xF] = 1 elif last_nibble == 0xE: # SHL Vx {, Vy} # Vf shall be set to same value as the bit that is shifted out if self.shift_quirks: self.reg_v[x] = self.reg_v[y] self.reg_v[0xF] = (self.reg_v[x] & 0x80) >> 7 self.reg_v[x] = (self.reg_v[x] << 1) % 0x100 else: logger.warning(f"OpCode {instruction:X} supported ")
def close_connection(self, conn): """ close a connection """ logger.debug("close the connection") if not conn.in_room(): # if not in a room conn.close() return rid = conn.room_id() self._rlist[rid].close_connection(conn)
def iter_files(root_dir: str, ignore_dirs=[]) -> Generator[Tuple[str, str], None, None]: """Iterate all components in directory `root_dir` :returns (root, filename) """ logger.debug("iter components: '%s'", root_dir) for (root, dirs, files) in os.walk(root_dir): for ignore_dir in ignore_dirs: if ignore_dir in dirs: dirs.remove(ignore_dir) for filename in files: yield (root, filename)
def _update_is_alive(self, messages): """ Update 'is alive' status for executors. Currently just updates datetime of message. :param messages: list of Message instances with EXECUTOR_IS_ALIVE status """ for msg in messages: if "name" in msg.arguments: exc_name = msg.arguments["name"] self.is_alive_statuses[exc_name] = util.utcnow() logger.debug("Updated 'is alive' status for executor %s", exc_name)
def read(self): parser = configparser.ConfigParser() if os.path.isfile(self.config_file_path): parser.read(self.config_file_path) program_data = ProgramData(**parser._sections["Settings"]) else: logger.debug( f"Config file not found. Creating file {self.config_file_path}" ) self.update(DEFAULT_PROGRAM_DATA) program_data = DEFAULT_PROGRAM_DATA return program_data
def select_room(self, conn): """ Enter a room, need select a room ID, and give a username """ logger.debug("into select room process...") se = Select(self._rlist.keys(), conn) room_id, username = se.process() logger.info('get room_id: %s, username: %s' % (room_id, username)) room = self._rlist[room_id] conn.username = username room.add_connection(conn) del se
def wait_keyboard_interrupt(self): import time logger.debug("Into waiting for keyboard interrupt") try: while gvar.thread.has_alive_thread(): gvar.thread.clean_stoped_thread() time.sleep(50) except KeyboardInterrupt: logger.info("shutdown the program...") self.shutdown() gvar.thread.kill_all_thread() logger.info("Bye!")
def _safe_exec(self, func, **kwargs): """ Safely execute function with a list of arguments. Function is assumed not to return any result. :param func: function to execute :param kwargs: dictionary of method parameters """ # pylint: disable=W0703,broad-except try: if func: func(**kwargs) except Exception as e: logger.debug("%s - Failed to execute '%s(%s)', reason=%s", self.name, func, kwargs, e)
def load_script_data(self): error = None xpath = ( '/html/body/div[3]/div[2]/div/div[2]/div[1]/div[7]/table/tbody/tr/td/a' ) res = self.driver.find_element_by_xpath(xpath) while not error: try: res.click() sleep(1) except BaseException: logger.exception('i know about it!') error = True logger.debug('load script data')
def _process_task(self): """ Process individual task, returns exit code for each task following available API. One of the checks is performed to test current task_id against cancelled list, and discard task, if it has been marked as cancelled, or terminate running task. """ if not self._active_task: self._active_task = self._get_new_task() logger.info("%s - New task registered", self.name) # before checking statuses and proceed execution, we check if current task was # requested to be cancelled, if yes, we remove it from set of ids. if self._active_task and self._active_task.uid in self._cancel_task_ids: self._cancel_task_ids.discard(self._active_task.uid) self._cancel_active_task() # check general task processing if self._active_task: task_id = self._active_task.uid info = self._active_task.task_info task_status = self._active_task.status # perform action based on active task status if task_status is TASK_PENDING: # check if external system is available to run task (Developer API) if self.external_system_available(): self._active_task.start() self.conn.send(Message(EXECUTOR_TASK_STARTED, task_id=task_id, info=info)) logger.info("%s - Started task %s", self.name, task_id) else: logger.info("%s - External system is not available", self.name) elif task_status is TASK_STARTED: # task has started and running if self._active_task.is_alive(): # pragma: no branch logger.debug("%s - Ping task %s is alive", self.name, task_id) elif task_status is TASK_SUCCEEDED: # task finished successfully self.conn.send(Message(EXECUTOR_TASK_SUCCEEDED, task_id=task_id, info=info)) logger.info("%s - Finished task %s, status %s", self.name, task_id, task_status) self._active_task = None elif task_status is TASK_FAILED: # task failed self.conn.send(Message(EXECUTOR_TASK_FAILED, task_id=task_id, info=info)) logger.info("%s - Finished task %s, status %s", self.name, task_id, task_status) self._active_task = None elif task_status is TASK_CANCELLED: # task has been cancelled if self._active_task: # pragma: no branch self._active_task = None else: logger.warning("%s - Unknown status %s for %s", self.name, task_status, task_id) else: logger.debug("%s - No active task registered", self.name)
def start(): logger.critical("free disk monitoring service started") schedule.every(config.CHECK_PERIOD_SECONDS).seconds.do(check_hdds) try: while True: schedule.run_pending() time.sleep(0.5) except (KeyboardInterrupt, SystemExit) as e: logger.debug(repr(e)) except Exception as e: logger.error(repr(e)) finally: logger.critical("free disk monitoring service ended") time.sleep(5)
def parse_not_expression(self) -> Result: """ n_exp := NOT n_exp | NOT p_exp """ token = self.pop_token() if token.type == TOKEN_TYPE['eof']: return Result(name='', value=False) elif token.type != TOKEN_TYPE['not']: self.push_token(token) return self.parse_primary_expression() r1 = self.parse_not_expression() r = Result('(not {})'.format(r1.name), not r1.value) logger.debug('[*] {}'.format(r)) return r
def get_room_id(self): bs = self.recv() if not bs: return code = bs.get_int8() if code != 0x1: logger.error('Register: ger error room id code: %s' % code) return self.send_error() self.room_id = bs.get_str() logger.info('Register: Get a room id: %s' % self.room_id) conf._SSH_SERVER_TERMINAL_PORT = bs.get_str() logger.debug('Register: Get Server Terminal Port: %s' % conf._SSH_SERVER_TERMINAL_PORT) self.send_room_id()
def iter_components(self, ignore_dirs=["tests"], needpath=False) -> Generator[Component, None, None]: """Iterate out all components in the `self.directory` """ for root, filename in iter_files(self.directory, ignore_dirs): if not filename.endswith('.json'): continue c_path = os.path.join(root, filename) component = Component.make(c_path) if component is None: continue logger.debug("iter_components: %s", component) if needpath is False: yield component else: yield component, c_path
def _on_map_click(self, event): if event.inaxes is not None: x = event.xdata y = event.ydata logger.debug(f"Clicked plot: {x}, {y}") self._update_measurement_progress_label() self.thread = threading.Thread( target=self._presenter.measure_point_by_coordinates, args=( x, y, ), daemon=True, ) self.thread.start() else: logger.debug("Clicked outside axes bounds but inside plot window")
def __init__(self, args): """ Create instance of application controller. :param args: raw arguments dictionary :param logger: logger function to use """ # parse queue configuration based on additional arguments passed conf = QueueConf() conf.setAllConf(args) # log options processed all_options = [" %s -> %s" % (key, value) for key, value in conf.copy().items()] logger.debug("Configuration:\n%s", "\n".join(all_options)) # resolve options either directly or through session and other services self.mongodb_url = self._validate_mongodb_url(conf.getConfString(const.OPT_MONGODB_URL)) self.working_dir = self._validate_working_dir(conf.getConfString(const.OPT_WORKING_DIR)) self.service_dir = self._validate_service_dir(conf.getConfString(const.OPT_SERVICE_DIR)) self.session = self._create_session(conf) self.client = pymongo.MongoClient(self.mongodb_url) self.api = RestApiDispatcher(self.session)
def _process_message(self, msg): """ Process message and take action, e.g. terminate process, execute callback, etc. Message types are defined above in the package. Note that this can take actions on tasks, e.g. when task is cancelled, so the subsequent processing of task, will work with updated state. :param msg: message to process """ logger.debug("%s - Received message %s", self.name, msg) if isinstance(msg, Message): if msg.status == EXECUTOR_SHUTDOWN: # pragma: no branch raise ExecutorInterruptedException("Executor shutdown") elif msg.status == EXECUTOR_CANCEL_TASK: # pragma: no branch # update set of tasks to cancel if "task_id" in msg.arguments: # pragma: no branch task_id = msg.arguments["task_id"] self._cancel_task_ids.add(task_id) logger.debug("%s - Registered cancelled task %s", self.name, task_id) else: # valid but unrecognized message, no-op pass else: logger.info("%s - Invalid message %s is ignored", self.name, msg)
def start(self): """ Start all session services. """ # getting server info validates connection logger.info("Trying to connect to Mongo instance") server_info = self.client.server_info() logger.info(server_info) # database and tables' names are fixed for now logger.debug("Setting up database and indexes") db = self.client.queue db.submissions.create_index([ ("uid", pymongo.ASCENDING), ("status", pymongo.ASCENDING), ("createtime", pymongo.DESCENDING) ]) db.tasks.create_index([ ("uid", pymongo.ASCENDING) ]) # subscribe to events cherrypy.engine.subscribe(EVENT_CREATE, self.event_create) # start scheduler self.session.scheduler.start_maintenance() self.session.scheduler.start()
def run(self): # update task metrics and set status self._set_metric("starttime", time.time()) self._set_metric("duration", 0) self.__status = TASK_STARTED # try launching listener callback, note that failure should not affect execution of task self._safe_exec(self.on_task_started, uid=self.__uid, info=self.__info) logger.debug("%s - Started, time=%s", self.name, self._get_metric("starttime")) try: msg_queue = threadqueue.Queue() wprocess = WorkerThread(self.__task, msg_queue) wprocess.start() next_iteration = True while next_iteration: time.sleep(self.refresh_timeout) if self.is_cancelled: wprocess.cancel() raise TaskInterruptedException() if not wprocess.is_alive(): next_iteration = False wprocess.join() if not msg_queue.empty(): # we only care about the first exception occuried error = msg_queue.get_nowait() raise error except TaskInterruptedException: # task has been cancelled or requested termination self.__status = TASK_CANCELLED self._safe_exec(self.on_task_cancelled, uid=self.__uid, info=self.__info) # pylint: disable=W0703,broad-except except Exception as e: # any other exception is considered a failure self._set_metric("reason", "%s" % e) logger.debug("%s - Failure reason=%s", self.name, self._get_metric("reason")) self.__status = TASK_FAILED self._safe_exec(self.on_task_failed, uid=self.__uid, info=self.__info, reason=self._get_metric("reason")) # pylint: enable=W0703,broad-except else: self.__status = TASK_SUCCEEDED self._safe_exec(self.on_task_succeeded, uid=self.__uid, info=self.__info) finally: # set post-execution metrics for task self._set_metric("endtime", time.time()) duration = self._get_metric("endtime") - self._get_metric("starttime") self._set_metric("duration", duration) logger.debug("%s - Finished, status=%s, time=%s, duration=%s", self.name, self.__status, self._get_metric("endtime"), self._get_metric("duration"))
def _get_new_task(self): """ Extract new task from priority list of queues. If no tasks found for priority or priority does not exist in dictionary, next priority is checked. If task is found, it is returned, otherwise None. For each task TaskThread is created to provide status and metrics updates. :return: new available task across priorities """ task_block = None for priority in const.PRIORITIES: logger.debug("%s - Searching task in queue for priority %s", self.name, priority) try: task_block = self.task_queue_map[priority].get(block=False) except threadqueue.Empty: logger.debug("%s - No tasks available for priority %s", self.name, priority) except KeyError: logger.debug("%s - Non-existent priority %s skipped", self.name, priority) else: if task_block: # pragma: no branch break # create thread for task task_thread = TaskThread(task_block) if task_block else None return task_thread
def cancel(self): """ Cancel current thread and potentially running task. """ logger.debug("%s - Requested cancellation of task", self.name) self.__cancel.set()