def get_env_vars() -> Dict[str, str]: """ Gets connection parameters from environment variables, fails and logs and error message if one of the environment variables is missing. :return: a dictionary containing the connection information """ try: return { "user": environ["POSTGRES_USER"], "password": environ["POSTGRES_PASSWORD"], "host": environ["POSTGRES_HOST"], "port": environ["POSTGRES_PORT"], "database": environ["POSTGRES_DB"], "twilio_account_sid": environ["TWILIO_ACCOUNT_SID"], "twilio_auth_token": environ["TWILIO_AUTH_TOKEN"], "twilio_supersim_sid": environ["TWILIO_SUPERSIM_SID"], } except KeyError as e: logger.error(f"Missing environment variable, make sure that " f"the following environment variables are defined: " f"POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, " f"POSTGRES_PORT, POSTGRES_DB, TWILIO_ACCOUNT_SID, " f"TWILIO_AUTH_TOKEN, TWILIO_SUPERSIM_SID") raise e
def get_params_from_env() -> Dict[str, str]: """ Gets connection parameters from environment variables, fails and logs and error message if one of the environment variables is missing. :return: a dictionary containing the connection information """ try: return { "user": environ["POSTGRES_USER"], "password": environ["POSTGRES_PASSWORD"], "host": environ["POSTGRES_HOST"], "port": environ["POSTGRES_PORT"], "database": environ["POSTGRES_DB"], } except KeyError: logger.error( f"Missing environment variable, make sure that " f"the following environment variables are defined: " f"POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, " f"POSTGRES_PORT, POSTGRES_DB" )
def init_tcp(self): self._tp = paramiko.Transport(self._socket, gss_kex=True) self._tp.set_gss_host(socket.getfqdn("")) self._tp.load_server_moduli() host_key = paramiko.RSAKey(filename=conf.SSH_SERVER_RSA_EKY) self._tp.add_server_key(host_key) server = Server() logger.debug('Create Server Class') try: self._tp.start_server(server=server) except paramiko.SSHException: logger.error("SSH negotiation failed.") gvar.manager.close_connection(self) exit() self._channel = self._tp.accept(200) if self._channel is None: logger.error('No channel') gvar.manager.close_connection(self) exit() logger.info('Authenticatied!')
def interpret_instruction(self, instruction: int) -> None: if instruction < 0x00: logger.error( f"Trying to pass a negative value {instruction:X} as instruction" ) return if instruction > 0xFFFF: logger.warning(f"Instruction {instruction:X} bigger than 8 bit," f" using {instruction % 0x10000:X}") instruction %= 0x10000 if instruction & 0xF000 == 0x0000: # Group 0 CLS, RET, SYS self.interpret_group_0(instruction) elif instruction & 0xF000 == 0x1000: # JP self.jump(instruction) elif instruction & 0xF000 == 0x2000: # CALL self.call(instruction) elif instruction & 0xF000 == 0x3000: # SE self.skip_if_equal(instruction) elif instruction & 0xF000 == 0x4000: # SNE self.skip_if_not_equal(instruction) elif instruction & 0xF000 == 0x5000: # SE self.skip_if_vx_equal_vy(instruction) elif instruction & 0xF000 == 0x6000: # LD (Vx, Byte) self.set_vx_to_kk(instruction) elif instruction & 0xF000 == 0x7000: # ADD (Vx, Byte) self.add_kk_to_vx(instruction) elif instruction & 0xF000 == 0x8000: # Arithmetic (Vx + Vy, Vx xor Vy etc.) self.interpret_group_8(instruction) elif instruction & 0xF000 == 0x9000: # SNE self.skip_if_vx_not_equal_vy(instruction) elif instruction & 0xF000 == 0xA000: # LD (Vx, Vy) self.set_index_to_nnn(instruction) elif instruction & 0xF000 == 0xB000: # JP V0 self.jump_with_offset(instruction) elif instruction & 0xF000 == 0xC000: # RND self.random(instruction) elif instruction & 0xF000 == 0xD000: # DRW self.draw(instruction) elif instruction & 0xF000 == 0xE000: # SKP, SKNP self.interpret_group_e(instruction) elif instruction & 0xF000 == 0xF000: # A lot of different LD variants + ADD (I, Vx) self.interpret_group_f(instruction)
def send(self, msg): try: self.channel.send(msg) return len(msg) except OSError: logger.error("server connection was closed.") gvar.manager.close_room()
def recv(self, msg): try: r = Protocol(msg) head = r.get_int8() if head != 0x1: return code = r.get_int8() if code == 0x0: self.get_request() elif code == 0x2: self.get_fin_room_id(r) elif code == 0xe: logger.error('Bad Room ID') self.send_error() elif code == 0xf: self.send_error() else: self.send_error() except StructError: self.send_error()
def send_error(self): """ if get error, then send error """ bs = self.get_protocol_head(0xf) self.send(bs) self.status = self.INIT_STATUS logger.error('Register Get an Error')
def close(self): try: for (_, room) in self._rlist.items(): room.close() except RuntimeError: logger.error("can't close room now, try again") time.sleep(0.2) self.close()
def calc_version(ver): r = re.match(r'v(\d+?)\.(\d+?)\.(\d+)', ver) if len(r.regs) != 4: logger.error("The version format is incorrect") return False _ = int(r.group(1)) * 10000 + int(r.group(2)) * 100 + int(r.group(3)) return _
def set_program_data(self, program_data: ProgramData): self.check_if_model_initialized() self.model.set_program_data(program_data) self.model.config_mgr.update(program_data) try: self.update_map() except Exception as e: logger.error(f"Exception occurred: {e}")
def make(cls, path: str): """Make a instance of 'Component' from JSON file """ try: with open(path, 'r', encoding="utf-8") as f: return cls(json.load(f)) except Exception as err: logger.error("'%s' make error: %s", cls.__name__, err) return None
def select_component_with(self, c_name) -> Optional[Dict]: try: with self._cnx.cursor() as cursor: sql = "SELECT `c_id`,`c_name`,`created_at` FROM `component` WHERE `c_name`=%s;" cursor.execute(sql, (c_name, )) return cursor.fetchone() except Exception as err: logger.error("Select component error: %s" % err) return None
def is_pressed(self, hex_key: int) -> bool: if hex_key > 0xF or hex_key < 0x0: logger.error(f"Invalid key {hex_key:X}") return False pygame_key = self.hex_pygame_key_map[hex_key] if pygame.key.get_pressed()[pygame_key]: return True else: return False
def get_connection(self, conn, port): if port == conf.SSH_SERVER_LISTENING_CLIENT_PORT: self.get_channel_connection(conn) elif port == conf.SSH_SERVER_LISTENING_TERMINIAL_PORT: self.get_control_connection(conn) else: logger.error('ssh listening port is not CONFIGURATION port') raise AttributeError
def request(self, url: str, **kwargs) -> Optional[Dict]: with self.request_manager_lock: r = self.request_manager_history.get(plain2md5(url), None) if not r is None: return r try: resp = requests.get(url, headers=self.headers, timeout=self.timeout, allow_redirects=self.allow_redirect, verify=False) except Exception as e: logger.error("request error: %s" % str(e)) return None script = [] meta = {} p = BeautifulSoup(resp.text, "html5lib") for data in p.find_all("script"): script_src = data.get("src") if script_src: script.append(script_src) for data in p.find_all("meta"): meta_name = data.get("name") meta_content = data.get("content", "") if meta_name: meta[meta_name] = meta_content title = p.find("title") if title: title = title.text else: title = "" raw_headers = '\n'.join('{}: {}'.format(k, v) for k, v in resp.headers.items()) resp = { "url": url, "body": resp.text, "headers": resp.headers, "status": resp.status_code, "script": script, "meta": meta, "title": title, "cookies": resp.cookies, "raw_cookies": resp.headers.get("set-cookie", ""), "raw_response": raw_headers + resp.text, "raw_headers": raw_headers, "md5": plain2md5(resp.content), } with self.request_manager_lock: self.request_manager_history[plain2md5(url)] = resp return resp
def start_listening(self, port): self._port = port self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.listener.bind((conf.SSH_SERVER_LISTENING_IP, self._port)) try: self.listener.listen() logger.info('SSH.start listening port:%s OK ...' % self._port) except: logger.error( 'SSH.start listening port:%s Error, Exit...' % self._port)
def _worker(): nonlocal _task_q while True: component_or_signal = _task_q.get() if component_or_signal == "QUIT": break component = component_or_signal try: self._process_check_matches_result(component) except Exception as err: logger.error("in _worker [%s] %s" % (component.name, err)) continue
def get_count(self) -> int: """get components count """ _count = 0 try: with self._cnx.cursor() as cursor: cursor.execute("SELECT count(_id) FROM `component`;") res = cursor.fetchone() if res: _count = res['count(_id)'] except Exception as err: logger.error("Select component error: %s" % err) return _count
def connection(self): try: # params: timeout = None means forever self.port = serial.Serial(conf.SERIAL_DEVICE, baudrate=conf.SERIAL_BAUDRATE, timeout=None, write_timeout=2, parity=serial.PARITY_NONE, bytesize=serial.EIGHTBITS, stopbits=serial.STOPBITS_ONE, xonxoff=False) except serial.serialutil.SerialException: logger.error( "can't open serial %s, Please check the COM is open and retry..." % conf.SERIAL_DEVICE) raise(serial.SerialException) self.port.flushInput() logger.info('serial port \'%s\' connection complete' % conf.SERIAL_DEVICE)
def start(): logger.critical("free disk monitoring service started") schedule.every(config.CHECK_PERIOD_SECONDS).seconds.do(check_hdds) try: while True: schedule.run_pending() time.sleep(0.5) except (KeyboardInterrupt, SystemExit) as e: logger.debug(repr(e)) except Exception as e: logger.error(repr(e)) finally: logger.critical("free disk monitoring service ended") time.sleep(5)
def create_ssh_server(self, port): listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) listener.bind((conf.SSH_SERVER_LISTENING_IP, port)) try: listener.listen() logger.info('SSH.start listening port:%s OK ...' % port) except: logger.error('SSH.start listening port:%s Error, Exit...' % port) var = SSHServer() var.manager = self # ! maybe not need var.listener = listener var._port = port var.thread_run() self.ssh_servers.append(var)
def run(self): while not self._thread_stop: ready = select.select([self.listener], [], [], None)[0] if self.listener._closed: return for _ in ready: # establish new TCP session try: _socket, addr = self.listener.accept() if self.manager: gvar.thread.function( target=self.manager.get_ssh_connection, args=(_socket, self._port), name='Got a SSH Connection') except Exception as e: logger.error('a bad socket %s ' % e)
def _create_raw_shapefile_from_csv(self, csv_path: str, input_shapefile_path: str): try: data = pd.read_csv(csv_path) with shp.Writer(input_shapefile_path, shapeType=shp.POLYGON) as w: w.field("ID", "N") for i, polygon in data.iterrows(): w.poly([[ [polygon["x1"], polygon["y1"]], [polygon["x2"], polygon["y2"]], [polygon["x3"], polygon["y3"]], [polygon["x4"], polygon["y4"]], [polygon["x1"], polygon["y1"]], ]]) w.record(polygon["id"]) except FileNotFoundError: logger.error(f"File {csv_path} not found!")
def get_room_id(self): bs = self.recv() if not bs: return code = bs.get_int8() if code != 0x1: logger.error('Register: ger error room id code: %s' % code) return self.send_error() self.room_id = bs.get_str() logger.info('Register: Get a room id: %s' % self.room_id) conf._SSH_SERVER_TERMINAL_PORT = bs.get_str() logger.debug('Register: Get Server Terminal Port: %s' % conf._SSH_SERVER_TERMINAL_PORT) self.send_room_id()
def select_components(self, limit: int = 500 ) -> Generator[List[Dict], None, None]: """select all componnents """ _count = self.get_count() if _count == 0: logger.warn("Remote database no components") return None for pre in range(0, _count, limit): try: with self._cnx.cursor() as cursor: sql = "SELECT `c_id`,`c_name`,`c_type`,`version`,`website`,`desc`,`producer`,`properties`,`matches`,`author`,`condition`, `implies`,`excludes`, `created_at` FROM `component` LIMIT %s,%s;" cursor.execute(sql, (pre, limit)) yield cursor.fetchall() except Exception as err: logger.error("Select component error: %s " % err) break
def get_new_version(self): try: resp = requests.get(url=banner.REPOSITORY_API_ADDRESS) except Exception as e: logger.error("network anomaly") return False if resp.status_code != 200: logger.info("unable to get a valid page") return False _ = str(resp.content.decode()) new_ver = re.search(r'"name":"(.*?)",', _).group(1) if not new_ver: logger.info("unable to get valid version information") return False return new_ver
def run(self): while not self._thread_stop: ready = select.select([self.listener], [], [], 2)[0] if self.listener._closed: self.thread_stop() continue for _ in ready: # establish new TCP session try: _socket, address = self.listener.accept() conn = ConnectionTelnet(_socket) self.manager.add_connection(conn) except Exception as e: logger.error('telnet listening catch a exception -> %s' % e) traceback.print_exc() self.close()
def insert_component(self, **component_info): now = datetime.datetime.now() component_info['created_at'] = now component_info['updated_at'] = now try: with self._cnx.cursor() as cursor: sql = "INSERT INTO `component` (`c_id`, `c_name`, `c_first`,`c_type`,`version`,`website`,`author`, `desc`, `producer`, `properties`,`matches`,`condition`, `implies`, `excludes`, `created_at`, `updated_at`) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" cursor.execute(sql, [ component_info.get(k) for k in ('c_id', 'c_name', 'c_first', 'c_type', 'version', 'website', 'author', 'desc', 'producer', 'properties', 'matches', 'condition', 'implies', 'excludes', 'created_at', 'updated_at') ]) self._cnx.commit() return True except Exception as err: logger.error("Insert component error: %s" % err) self._cnx.rollback() return False
def update_component_with(self, c_name, **component_info) -> bool: now = datetime.datetime.now() component_info['updated_at'] = now component_info['c_name'] = c_name try: with self._cnx.cursor() as cursor: sql = "UPDATE `component` SET `c_type`=%s, `author`=%s, `desc`=%s, `producer`=%s, `properties`=%s,`matches`=%s, `condition`=%s, `implies`=%s, `excludes`=%s, `updated_at`=%s WHERE `c_name`=%s;" cursor.execute(sql, [ component_info.get(k) for k in ('c_type', 'author', 'desc', 'producer', 'properties', 'matches', 'condition', 'implies', 'excludes', 'updated_at', 'c_name') ]) self._cnx.commit() return True except Exception as err: logger.error("Update component error: %s" % err) self._cnx.rollback() return False
def check_conf(self): if not conf.SERIAL_DEVICE: logger.info( 'No Serial Port specified in the configuration, find Serial Ports...') plist = port_list.comports() if not plist: logger.error( "can't found any serial port, Please check Serial Port") return conf.SERIAL_DEVICE = plist[0].device logger.info('Specifiy Serial Port > %s' % conf.SERIAL_DEVICE) if not conf.SERIAL_BAUDRATE: logger.error( "No Serial Baudrate specified in the configuration, Set to 9600") conf.SERIAL_BAUDRATE = 9600 logger.info('serial port configraution check finished') return