def send_sms(num, msg):
if not SmsWrapper.inited():
log.error(
"[Send SMS] Not able to send SMS to '%s' as SMS module is down" %
num)
return None
return SmsWrapper.sms.send_sms(num=num, msg=msg)
def to_mail(gtk, cves, smtp, sender, password):
content = format_content(cves)
receivers = load_local_receivers()
if gtk:
log.info('[邮件] 正在通过 Github Actions 推送威胁情报...')
recvs = load_issue_receivers(gtk)
recvs.update(receivers)
to_cache(','.join(recvs), MAIL_RECV_CACHE)
to_cache(content, MAIL_CONTENT_CACHE)
else:
log.info('[邮件] 正在推送威胁情报...')
email = MIMEText(content, 'html', env.CHARSET) # 以 html 格式发送邮件内容
email['From'] = sender
email['To'] = ', '.join(receivers) # 此处收件人列表必须为逗号分隔的 str
log.info('[邮件] 收件人清单： %s' % receivers)
subject = '威胁情报播报'
email['Subject'] = Header(subject, 'utf-8')
try:
smtpObj = smtplib.SMTP(smtp)
smtpObj.login(sender, password)
smtpObj.sendmail(sender, receivers,
email.as_string()) # 此处收件人列表必须为 list
log.info('[邮件] 推送威胁情报成功')
except:
log.error('[邮件] 推送威胁情报失败')
def process_request(self, goip):
if not self.has_request():
log.info("[ProcessRequest] No request to process")
return
update = self.request.update
context = self.request.context
try:
threading.Thread(target=start_over, args=(
update,
context,
)).start()
send_bot_msg(update, context, msg="Виконую запит")
result = self.request.process(goip)
if result:
bot.send(result)
send_bot_msg(update, context, msg="Тринь, ісполнєно!")
except Exception as e:
try:
send_bot_msg(update, context, msg="Трапилась помилка")
log.error(
"[Personal bot] Exception while processing request: %s" %
e)
except Exception as e1:
log.error(
"[Personal bot] Exception while handling exception: %s\noriginal exception: %s"
% (e1, e))
finally:
self.request = None
def cves(self):
log.info('++++++++++++++++++++++++++++++++++++++++++++')
log.info('正在获取 [%s] 威胁情报...' % self.NAME_CH())
old_cves = self.load_cache()
try:
new_cves = self.get_cves()
except:
new_cves = []
log.error('获取 [%s] 威胁情报异常' % self.NAME_CH())
dao = TCvesDao()
sdbc = SqliteSDBC(env.DB_PATH)
conn = sdbc.conn()
_cves = []
for cve in new_cves:
if cve.MD5() not in old_cves:
_cves.append(cve)
self.to_cache(cve)
self.to_db(conn, dao, cve)
sdbc.close()
log.info('得到 [%s] 最新威胁情报 [%s] 条' % (self.NAME_CH(), len(_cves)))
log.info('--------------------------------------------')
return _cves
async def predict(request: EngineRequest):
log.info('Starting request...')
try:
# Validate request
if request.secret_access == env.get_secret_access():
# Open image
image = np.array(Image.open(request.img).convert('RGB'))
# Run inference
result, error_message = await run(net, image, request.to_remove,
request.color_removal)
if result:
# Save image
tmp_file_name = os.path.join(TMP_FOLDER,
'{}.png'.format(uuid.uuid4()))
result.save(tmp_file_name)
# Return
log.info('Image saved in: {}'.format(tmp_file_name))
return dict(error=False,
img=tmp_file_name,
message=EXAMPLE_MESSAGE_SUCCESS)
else:
return dict(error=True, img=None, message=error_message)
else:
return dict(error=True, img=None, message='Unauthorized.')
except Exception as e:
error_message = 'Error on request: [{}]'.format(e)
log.error(error_message)
log.exception(e)
return dict(error=True, img=None, message=error_message)
def write_json_file(path, content):
try:
log.info('Opening ' + path + ' file ...')
with open(path, 'w') as outfile:
json.dump(content, outfile, ensure_ascii=False)
log.info(path + ' file writed.')
except IOError as err: # whatever reader errors you care about
log.error('|IOError| - File not found or couldn\'t be open to write')
def read_json_file(path):
try:
log.info('Reading ' + path + ' file ...')
with open(path, 'r') as f:
log.info(path + ' file opened.')
return json.load(f)
except IOError as err: # whatever reader errors you care about
log.error('|IOError| - File not found or couldn\'t be open')
def init_browser(self, pwd=None):
try:
BrowserWrapper.init(self.url, self.uname, pwd or self.pwd)
except NotLoggedIn as e:
log.error("[Init browser] Error in init_browser: %s" % e)
if pwd != DEFAULT_GOIP_PWD:
log.warning("[Init Browser] Logging in using default password")
BrowserWrapper.init(self.url, self.uname, DEFAULT_GOIP_PWD)
BrowserWrapper.b.driver.refresh()
def init(cls, url, uname, pwd, notify_module_is_up=False):
cls.kill()
try:
cls.sms = Sms(url,
uname,
pwd,
notify_module_is_up=notify_module_is_up)
cls._inited = True
except Exception as e:
log.error(e)
def send_bot_msg(update, context, msg, reply_markup=None, noedit=False):
if noedit and update.message:
update.message.reply_text(text=msg, reply_markup=reply_markup)
return
if update.callback_query:
update.callback_query.edit_message_text(text=msg,
reply_markup=reply_markup)
elif update.message:
update.message.reply_text(text=msg, reply_markup=reply_markup)
else:
log.error("[Personal bot] Unable to send the message: %s" % msg)
def auto_commit():
log.info('正在提交变更...')
try:
repo = git.Repo(env.PRJ_DIR)
repo.git.add('*')
repo.git.commit(m='[Threat-Broadcast] %s' % time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
repo.git.push()
log.info('提交变更成功')
except:
log.error('提交变更失败')
def get_dataframe_from_json(content):
try:
log.info('Generating DataFrame...')
df = pd.DataFrame(data=content['articles'])
log.info('Done')
print(get_columns_name(df))
df.info(memory_usage='deep')
return df
except ValueError as err:
log.error('|DATAFRAME.py| - Can\'t create pandas dataframe')
log.error(err)
def command_func(update, context, *args, **kwargs):
if update is None or context is None:
return func(update, context, *args, **kwargs)
username = update.effective_user.username
if username not in ALLOWED_USERS:
log.error(
"[Personal bot] Unauthorized access denied for {}.".format(
username))
send_bot_msg(update, context, msg='Unauthorised.')
return # quit function
return func(update, context, *args, **kwargs)
def load_issue_receivers(gtk):
recvs = set()
try:
issues = _git.query_issues(gtk)
for issue in issues :
ptn = re.compile(r'([a-zA-Z0-9_-]+@[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)+)')
emails = ptn.findall(issue)
for email in emails:
recvs.add(email[0])
except:
log.error('获取 Issue 的邮箱失败（国内 GraphQL 接口不稳定）')
return recvs
def conn(self):
"""
连接到数据库
:return: 数据库连接（失败返回 None）
"""
if not self._conn:
try:
self._conn = sqlite3.connect(database=self.dbpath)
self._conn.text_factory = str
except:
log.error("连接数据库 [%s] 失败" % self.dbpath)
return self._conn
def commit(self):
"""
提交事务
:return: 是否提交成功
"""
is_ok = False
if self._conn:
try:
self._conn.commit()
is_ok = True
except:
log.error("提交事务到数据库 [%s] 失败" % self.dbpath)
return is_ok
def query_srcs(conn):
sql = 'select %s from %s group by %s' % (TCves.s_src, TCves.table_name, TCves.s_src)
srcs = []
try:
cursor = conn.cursor()
cursor.execute(sql)
rows = cursor.fetchall()
for row in rows:
srcs.append(row[0])
cursor.close()
except:
log.error("从表 [%s] 查询数据失败" % TCves.table_name)
return srcs
def auto_commit():
log.info("正在提交变更...")
try:
repo = git.Repo(config.PRJ_DIR)
repo.git.add("*")
repo.git.commit(
m="[Threat-Broadcast] %s" %
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))
repo.git.push()
log.info("提交变更成功")
except:
log.error("提交变更失败")
def close(self, err_log=False):
if not self.driver:
return
log.info("[Browser] Close")
try:
self.driver.close() # close the current page
self.driver.service.process.send_signal(
signal.SIGTERM) # kill the specific phantomjs child proc
self.driver.quit() # quit the node proc
except Exception as e:
if err_log:
log.error("[Browser] Close exception : {}".format(e))
self.driver = None
def close(self, force=False):
if not self._all_processes:
return
log.info("[Close] Disconnecting the SMPP client")
try:
self.client.disconnect()
except Exception as e:
log.error(e)
if not force:
sleep(2)
log.info("[Close] Terminating processes for SMS monitoring")
for process in self._all_processes:
process.terminate()
def close(self):
"""
断开数据库连接
:return: 是否断开成功
"""
is_ok = False
if self._conn:
try:
self._conn.close()
self._conn = None
is_ok = True
except:
log.error("断开数据库 [%s] 失败" % self.dbpath)
return is_ok
def query_srcs(conn):
sql = "SELECT %s FROM %s GROUP BY %s" % (TCves.s_src, TCves.table_name,
TCves.s_src)
srcs = []
try:
cursor = conn.cursor()
cursor.execute(sql)
rows = cursor.fetchall()
for row in rows:
srcs.append(row[0])
cursor.close()
except:
log.error("从表 [%s] 查询数据失败" % TCves.table_name)
return srcs
def count(self, conn):
"""
统计行数
:param conn: 数据库连接
:return: 表行数
"""
cnt = 0
try:
cursor = conn.cursor()
cursor.execute(self.SQL_COUNT)
cnt = cursor.fetchone()[0]
cursor.close()
except:
log.error("统计表 [%s] 行数失败" % self.TABLE_NAME)
return cnt
def wrapper(*args):
params = foo(*args)
if tp is 'create':
params[2].entries.add().CopyFrom(params[0])
flag = args[1].set_state(
{params[1]: params[2].SerializeToString()})
if tp is 'update':
container_data = args[1].get_state([params[1]])
if len(container_data) < 1:
log.error('state has no this address')
params[2].ParseFromString(container_data[0].data)
params[2].entries.add().CopyFrom(params[0])
flag = args[1].set_state(
{params[1]: params[2].SerializeToString()})
return flag
def goip_monitor(self):
# we couldn't afford sleep(600) because we are working with browser in the single thread
# instead - just skip method' body till the sleep time is elapsed
if self.goip_slept_at and not passed_more_that_sec(
self.goip_slept_at, GOIP_MONITOR_SLEEP_SECONDS):
return True
cdr_started = BrowserWrapper.b.by_id("l1_cdrt").text.strip()
if cdr_started.startswith("1970-01"):
log.error(
"[GoipMonitor] Have internal GoIP issue (1970 year at clock).")
if vs.last_date_cdr_restart() == current_time().date(
): # if we had the same problem today
bot.send(
"Переналаштовую дзвонилку бо вона ґеґнула (1970 рік надворі)!"
)
self.reset_and_restore()
else:
vs.set_last_date_cdr_restart(current_time().date(
)) # if this is the first problem occurrence
bot.send(
"Перезавантажую дзвонилку бо вона знову ні-гугу (1970 рік надворі)!"
)
self.reboot()
# open the 'Status' tab again to support the logic in rest of the cycle
BrowserWrapper.b.open_menu("Status")
# just waiting for the fix to be applied. Nothing could be done now
return False
# reason for the status check is doing fix only if GoIP problem persists for >1 cycle
goip_is_working = self.statuses_ok()
if not goip_is_working and vs.last_reg_status(
) == self.voip_connection_status:
self.reset_and_restore()
# open the 'Status' tab again to support the logic in rest of the cycle
BrowserWrapper.b.open_menu("Status")
self.voip_connection_status = vs.last_reg_status()
self.goip_slept_at = current_time(
) # we have this in-memory var to decrease amt of calls to the DB
vs.set_monitor_slept_at(
self.goip_slept_at
) # we are using this value as heartbeat for the whole GoIP monitor
log.info("[GoipMonitor] Sleeping for %d sec..." %
GOIP_MONITOR_SLEEP_SECONDS)
if not goip_is_working: # just waiting for the fix to be applied. Nothing could be done now
log.error(
"[GoipMonitor] Patient is not ok. Will check again soon.")
return False
return True
def query_cves(conn, src, limit):
dao = TCvesDao()
where = "and %s = '%s' order by %s desc limit %d" % (TCves.s_src, src, TCves.s_time, limit)
sql = TCvesDao.SQL_SELECT + where
beans = []
try:
cursor = conn.cursor()
cursor.execute(sql)
rows = cursor.fetchall()
for row in rows:
bean = dao._to_bean(row)
beans.append(bean)
cursor.close()
except:
log.error("从表 [%s] 查询数据失败" % TCves.table_name)
return beans
def truncate(self, conn):
"""
清空表
:param conn: 数据库连接
:return: 是否清空成功
"""
is_ok = False
try:
cursor = conn.cursor()
cursor.execute(self.SQL_TRUNCATE)
conn.commit()
cursor.close()
is_ok = True
except:
log.error("清空表 [%s] 失败" % self.TABLE_NAME)
return is_ok
def insert(self, conn, bean):
"""
插入单条数据
:param conn: 数据库连接
:param bean: 数据模型实例
:return: 是否插入成功
"""
is_ok = False
try:
cursor = conn.cursor()
params = bean.params()
cursor.execute(self.SQL_INSERT, params)
conn.commit()
cursor.close()
is_ok = True
except:
log.error("插入数据到表 [%s] 失败" % self.TABLE_NAME)
return is_ok
def delete(self, conn, wheres={}):
"""
删除数据
:param conn: 数据库连接
:param wheres: 条件键值对, 要求键值包含操作符，如： { 'column1 like': 'xyz', 'column2 =': 'abc' }
:return: 是否删除成功
"""
is_ok = False
try:
cursor = conn.cursor()
sql = self._append(self.SQL_DELETE, wheres.keys())
cursor.execute(sql, wheres.values())
conn.commit()
cursor.close()
is_ok = True
except:
log.error("从表 [%s] 删除数据失败" % self.TABLE_NAME)
return is_ok
def query_one(self, conn, wheres={}):
"""
查询表中一条数据
:param conn: 数据库连接
:param wheres: 条件键值对, 要求键值包含操作符，如： { 'column1 like': 'xyz', 'column2 =': 'abc' }
:return: 数据模型实例队列（若多个满足则返回第 1 个，没有满足则返回 None）
"""
bean = None
try:
cursor = conn.cursor()
sql = self._append(self.SQL_SELECT, wheres.keys())
cursor.execute(sql, wheres.values())
row = cursor.fetchone()
bean = self._to_bean(row)
conn.commit()
cursor.close()
except:
log.error("从表 [%s] 查询数据失败" % self.TABLE_NAME)
return bean
