def get_env_vars() -> Dict[str, str]:
"""
Gets connection parameters from environment variables, fails and logs and error message
if one of the environment variables is missing.
:return: a dictionary containing the connection information
"""
try:
return {
"user": environ["POSTGRES_USER"],
"password": environ["POSTGRES_PASSWORD"],
"host": environ["POSTGRES_HOST"],
"port": environ["POSTGRES_PORT"],
"database": environ["POSTGRES_DB"],
"twilio_account_sid": environ["TWILIO_ACCOUNT_SID"],
"twilio_auth_token": environ["TWILIO_AUTH_TOKEN"],
"twilio_supersim_sid": environ["TWILIO_SUPERSIM_SID"],
}
except KeyError as e:
logger.error(f"Missing environment variable, make sure that "
f"the following environment variables are defined: "
f"POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, "
f"POSTGRES_PORT, POSTGRES_DB, TWILIO_ACCOUNT_SID, "
f"TWILIO_AUTH_TOKEN, TWILIO_SUPERSIM_SID")
raise e
def get_params_from_env() -> Dict[str, str]:
"""
Gets connection parameters from environment variables, fails and logs and error message
if one of the environment variables is missing.
:return: a dictionary containing the connection information
"""
try:
return {
"user": environ["POSTGRES_USER"],
"password": environ["POSTGRES_PASSWORD"],
"host": environ["POSTGRES_HOST"],
"port": environ["POSTGRES_PORT"],
"database": environ["POSTGRES_DB"],
}
except KeyError:
logger.error(
f"Missing environment variable, make sure that "
f"the following environment variables are defined: "
f"POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, "
f"POSTGRES_PORT, POSTGRES_DB"
)
def init_tcp(self):
self._tp = paramiko.Transport(self._socket, gss_kex=True)
self._tp.set_gss_host(socket.getfqdn(""))
self._tp.load_server_moduli()
host_key = paramiko.RSAKey(filename=conf.SSH_SERVER_RSA_EKY)
self._tp.add_server_key(host_key)
server = Server()
logger.debug('Create Server Class')
try:
self._tp.start_server(server=server)
except paramiko.SSHException:
logger.error("SSH negotiation failed.")
gvar.manager.close_connection(self)
exit()
self._channel = self._tp.accept(200)
if self._channel is None:
logger.error('No channel')
gvar.manager.close_connection(self)
exit()
logger.info('Authenticatied!')
def interpret_instruction(self, instruction: int) -> None:
if instruction < 0x00:
logger.error(
f"Trying to pass a negative value {instruction:X} as instruction"
)
return
if instruction > 0xFFFF:
logger.warning(f"Instruction {instruction:X} bigger than 8 bit,"
f" using {instruction % 0x10000:X}")
instruction %= 0x10000
if instruction & 0xF000 == 0x0000:
# Group 0 CLS, RET, SYS
self.interpret_group_0(instruction)
elif instruction & 0xF000 == 0x1000:
# JP
self.jump(instruction)
elif instruction & 0xF000 == 0x2000:
# CALL
self.call(instruction)
elif instruction & 0xF000 == 0x3000:
# SE
self.skip_if_equal(instruction)
elif instruction & 0xF000 == 0x4000:
# SNE
self.skip_if_not_equal(instruction)
elif instruction & 0xF000 == 0x5000:
# SE
self.skip_if_vx_equal_vy(instruction)
elif instruction & 0xF000 == 0x6000:
# LD (Vx, Byte)
self.set_vx_to_kk(instruction)
elif instruction & 0xF000 == 0x7000:
# ADD (Vx, Byte)
self.add_kk_to_vx(instruction)
elif instruction & 0xF000 == 0x8000:
# Arithmetic (Vx + Vy, Vx xor Vy etc.)
self.interpret_group_8(instruction)
elif instruction & 0xF000 == 0x9000:
# SNE
self.skip_if_vx_not_equal_vy(instruction)
elif instruction & 0xF000 == 0xA000:
# LD (Vx, Vy)
self.set_index_to_nnn(instruction)
elif instruction & 0xF000 == 0xB000:
# JP V0
self.jump_with_offset(instruction)
elif instruction & 0xF000 == 0xC000:
# RND
self.random(instruction)
elif instruction & 0xF000 == 0xD000:
# DRW
self.draw(instruction)
elif instruction & 0xF000 == 0xE000:
# SKP, SKNP
self.interpret_group_e(instruction)
elif instruction & 0xF000 == 0xF000:
# A lot of different LD variants + ADD (I, Vx)
self.interpret_group_f(instruction)
def send(self, msg):
try:
self.channel.send(msg)
return len(msg)
except OSError:
logger.error("server connection was closed.")
gvar.manager.close_room()
def recv(self, msg):
try:
r = Protocol(msg)
head = r.get_int8()
if head != 0x1:
return
code = r.get_int8()
if code == 0x0:
self.get_request()
elif code == 0x2:
self.get_fin_room_id(r)
elif code == 0xe:
logger.error('Bad Room ID')
self.send_error()
elif code == 0xf:
self.send_error()
else:
self.send_error()
except StructError:
self.send_error()
def send_error(self):
"""
if get error, then send error
"""
bs = self.get_protocol_head(0xf)
self.send(bs)
self.status = self.INIT_STATUS
logger.error('Register Get an Error')
def close(self):
try:
for (_, room) in self._rlist.items():
room.close()
except RuntimeError:
logger.error("can't close room now, try again")
time.sleep(0.2)
self.close()
def calc_version(ver):
r = re.match(r'v(\d+?)\.(\d+?)\.(\d+)', ver)
if len(r.regs) != 4:
logger.error("The version format is incorrect")
return False
_ = int(r.group(1)) * 10000 + int(r.group(2)) * 100 + int(r.group(3))
return _
def set_program_data(self, program_data: ProgramData):
self.check_if_model_initialized()
self.model.set_program_data(program_data)
self.model.config_mgr.update(program_data)
try:
self.update_map()
except Exception as e:
logger.error(f"Exception occurred: {e}")
def make(cls, path: str):
"""Make a instance of 'Component' from JSON file
"""
try:
with open(path, 'r', encoding="utf-8") as f:
return cls(json.load(f))
except Exception as err:
logger.error("'%s' make error: %s", cls.__name__, err)
return None
def select_component_with(self, c_name) -> Optional[Dict]:
try:
with self._cnx.cursor() as cursor:
sql = "SELECT `c_id`,`c_name`,`created_at` FROM `component` WHERE `c_name`=%s;"
cursor.execute(sql, (c_name, ))
return cursor.fetchone()
except Exception as err:
logger.error("Select component error: %s" % err)
return None
def is_pressed(self, hex_key: int) -> bool:
if hex_key > 0xF or hex_key < 0x0:
logger.error(f"Invalid key {hex_key:X}")
return False
pygame_key = self.hex_pygame_key_map[hex_key]
if pygame.key.get_pressed()[pygame_key]:
return True
else:
return False
def get_connection(self, conn, port):
if port == conf.SSH_SERVER_LISTENING_CLIENT_PORT:
self.get_channel_connection(conn)
elif port == conf.SSH_SERVER_LISTENING_TERMINIAL_PORT:
self.get_control_connection(conn)
else:
logger.error('ssh listening port is not CONFIGURATION port')
raise AttributeError
def request(self, url: str, **kwargs) -> Optional[Dict]:
with self.request_manager_lock:
r = self.request_manager_history.get(plain2md5(url), None)
if not r is None:
return r
try:
resp = requests.get(url,
headers=self.headers,
timeout=self.timeout,
allow_redirects=self.allow_redirect,
verify=False)
except Exception as e:
logger.error("request error: %s" % str(e))
return None
script = []
meta = {}
p = BeautifulSoup(resp.text, "html5lib")
for data in p.find_all("script"):
script_src = data.get("src")
if script_src:
script.append(script_src)
for data in p.find_all("meta"):
meta_name = data.get("name")
meta_content = data.get("content", "")
if meta_name:
meta[meta_name] = meta_content
title = p.find("title")
if title:
title = title.text
else:
title = ""
raw_headers = '\n'.join('{}: {}'.format(k, v)
for k, v in resp.headers.items())
resp = {
"url": url,
"body": resp.text,
"headers": resp.headers,
"status": resp.status_code,
"script": script,
"meta": meta,
"title": title,
"cookies": resp.cookies,
"raw_cookies": resp.headers.get("set-cookie", ""),
"raw_response": raw_headers + resp.text,
"raw_headers": raw_headers,
"md5": plain2md5(resp.content),
}
with self.request_manager_lock:
self.request_manager_history[plain2md5(url)] = resp
return resp
def start_listening(self, port):
self._port = port
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.listener.bind((conf.SSH_SERVER_LISTENING_IP, self._port))
try:
self.listener.listen()
logger.info('SSH.start listening port:%s OK ...' % self._port)
except:
logger.error(
'SSH.start listening port:%s Error, Exit...' % self._port)
def _worker():
nonlocal _task_q
while True:
component_or_signal = _task_q.get()
if component_or_signal == "QUIT":
break
component = component_or_signal
try:
self._process_check_matches_result(component)
except Exception as err:
logger.error("in _worker [%s] %s" % (component.name, err))
continue
def get_count(self) -> int:
"""get components count
"""
_count = 0
try:
with self._cnx.cursor() as cursor:
cursor.execute("SELECT count(_id) FROM `component`;")
res = cursor.fetchone()
if res:
_count = res['count(_id)']
except Exception as err:
logger.error("Select component error: %s" % err)
return _count
def connection(self):
try:
# params: timeout = None means forever
self.port = serial.Serial(conf.SERIAL_DEVICE, baudrate=conf.SERIAL_BAUDRATE, timeout=None, write_timeout=2,
parity=serial.PARITY_NONE, bytesize=serial.EIGHTBITS, stopbits=serial.STOPBITS_ONE, xonxoff=False)
except serial.serialutil.SerialException:
logger.error(
"can't open serial %s, Please check the COM is open and retry..." % conf.SERIAL_DEVICE)
raise(serial.SerialException)
self.port.flushInput()
logger.info('serial port \'%s\' connection complete' %
conf.SERIAL_DEVICE)
def start():
logger.critical("free disk monitoring service started")
schedule.every(config.CHECK_PERIOD_SECONDS).seconds.do(check_hdds)
try:
while True:
schedule.run_pending()
time.sleep(0.5)
except (KeyboardInterrupt, SystemExit) as e:
logger.debug(repr(e))
except Exception as e:
logger.error(repr(e))
finally:
logger.critical("free disk monitoring service ended")
time.sleep(5)
def create_ssh_server(self, port):
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind((conf.SSH_SERVER_LISTENING_IP, port))
try:
listener.listen()
logger.info('SSH.start listening port:%s OK ...' % port)
except:
logger.error('SSH.start listening port:%s Error, Exit...' % port)
var = SSHServer()
var.manager = self # ! maybe not need
var.listener = listener
var._port = port
var.thread_run()
self.ssh_servers.append(var)
def run(self):
while not self._thread_stop:
ready = select.select([self.listener], [], [], None)[0]
if self.listener._closed:
return
for _ in ready: # establish new TCP session
try:
_socket, addr = self.listener.accept()
if self.manager:
gvar.thread.function(
target=self.manager.get_ssh_connection, args=(_socket, self._port), name='Got a SSH Connection')
except Exception as e:
logger.error('a bad socket %s ' % e)
def _create_raw_shapefile_from_csv(self, csv_path: str,
input_shapefile_path: str):
try:
data = pd.read_csv(csv_path)
with shp.Writer(input_shapefile_path, shapeType=shp.POLYGON) as w:
w.field("ID", "N")
for i, polygon in data.iterrows():
w.poly([[
[polygon["x1"], polygon["y1"]],
[polygon["x2"], polygon["y2"]],
[polygon["x3"], polygon["y3"]],
[polygon["x4"], polygon["y4"]],
[polygon["x1"], polygon["y1"]],
]])
w.record(polygon["id"])
except FileNotFoundError:
logger.error(f"File {csv_path} not found!")
def get_room_id(self):
bs = self.recv()
if not bs:
return
code = bs.get_int8()
if code != 0x1:
logger.error('Register: ger error room id code: %s' % code)
return self.send_error()
self.room_id = bs.get_str()
logger.info('Register: Get a room id: %s' % self.room_id)
conf._SSH_SERVER_TERMINAL_PORT = bs.get_str()
logger.debug('Register: Get Server Terminal Port: %s' %
conf._SSH_SERVER_TERMINAL_PORT)
self.send_room_id()
def select_components(self,
limit: int = 500
) -> Generator[List[Dict], None, None]:
"""select all componnents
"""
_count = self.get_count()
if _count == 0:
logger.warn("Remote database no components")
return None
for pre in range(0, _count, limit):
try:
with self._cnx.cursor() as cursor:
sql = "SELECT `c_id`,`c_name`,`c_type`,`version`,`website`,`desc`,`producer`,`properties`,`matches`,`author`,`condition`, `implies`,`excludes`, `created_at` FROM `component` LIMIT %s,%s;"
cursor.execute(sql, (pre, limit))
yield cursor.fetchall()
except Exception as err:
logger.error("Select component error: %s " % err)
break
def get_new_version(self):
try:
resp = requests.get(url=banner.REPOSITORY_API_ADDRESS)
except Exception as e:
logger.error("network anomaly")
return False
if resp.status_code != 200:
logger.info("unable to get a valid page")
return False
_ = str(resp.content.decode())
new_ver = re.search(r'"name":"(.*?)",', _).group(1)
if not new_ver:
logger.info("unable to get valid version information")
return False
return new_ver
def run(self):
while not self._thread_stop:
ready = select.select([self.listener], [], [], 2)[0]
if self.listener._closed:
self.thread_stop()
continue
for _ in ready: # establish new TCP session
try:
_socket, address = self.listener.accept()
conn = ConnectionTelnet(_socket)
self.manager.add_connection(conn)
except Exception as e:
logger.error('telnet listening catch a exception -> %s' %
e)
traceback.print_exc()
self.close()
def insert_component(self, **component_info):
now = datetime.datetime.now()
component_info['created_at'] = now
component_info['updated_at'] = now
try:
with self._cnx.cursor() as cursor:
sql = "INSERT INTO `component` (`c_id`, `c_name`, `c_first`,`c_type`,`version`,`website`,`author`, `desc`, `producer`, `properties`,`matches`,`condition`, `implies`, `excludes`, `created_at`, `updated_at`) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
cursor.execute(sql, [
component_info.get(k)
for k in ('c_id', 'c_name', 'c_first', 'c_type', 'version',
'website', 'author', 'desc', 'producer',
'properties', 'matches', 'condition', 'implies',
'excludes', 'created_at', 'updated_at')
])
self._cnx.commit()
return True
except Exception as err:
logger.error("Insert component error: %s" % err)
self._cnx.rollback()
return False
def update_component_with(self, c_name, **component_info) -> bool:
now = datetime.datetime.now()
component_info['updated_at'] = now
component_info['c_name'] = c_name
try:
with self._cnx.cursor() as cursor:
sql = "UPDATE `component` SET `c_type`=%s, `author`=%s, `desc`=%s, `producer`=%s, `properties`=%s,`matches`=%s, `condition`=%s, `implies`=%s, `excludes`=%s, `updated_at`=%s WHERE `c_name`=%s;"
cursor.execute(sql, [
component_info.get(k)
for k in ('c_type', 'author', 'desc', 'producer',
'properties', 'matches', 'condition', 'implies',
'excludes', 'updated_at', 'c_name')
])
self._cnx.commit()
return True
except Exception as err:
logger.error("Update component error: %s" % err)
self._cnx.rollback()
return False
def check_conf(self):
if not conf.SERIAL_DEVICE:
logger.info(
'No Serial Port specified in the configuration, find Serial Ports...')
plist = port_list.comports()
if not plist:
logger.error(
"can't found any serial port, Please check Serial Port")
return
conf.SERIAL_DEVICE = plist[0].device
logger.info('Specifiy Serial Port > %s' % conf.SERIAL_DEVICE)
if not conf.SERIAL_BAUDRATE:
logger.error(
"No Serial Baudrate specified in the configuration, Set to 9600")
conf.SERIAL_BAUDRATE = 9600
logger.info('serial port configraution check finished')
return
