def send_mail(content):
smtp_config = setting_col.find_one({'key': 'mail'})
receivers = [data.get('mail') for data in notice_col.find({})]
try:
if mail_notice(smtp_config, receivers, content):
logger.info('邮件发送成功')
else:
logger.critical('Error: 无法发送邮件')
except smtplib.SMTPException as error:
logger.critical('Error: 无法发送邮件 {}'.format(error))
def run(self):
"""
Run all of the registered functions.
"""
self._run = True
for app_name, f in self._init_funcs.iteritems():
try:
f()
except:
logger.critical(app_name + ' server side failed to initialize with the exception:')
logger.critical(traceback.format_exc())
def send_mail(content):
smtp_config = setting_col.find_one({'key': 'mail'})
receivers = [data.get('mail') for data in notice_col.find({})]
elementcmd = "kmg imu zengshuai \'[GitHub警告]<br />" + content + "\'"
try:
if mail_notice(smtp_config, receivers, content):
logger.info('邮件发送成功')
else:
os.system("kmg imu zengshuai 'haweye email Error'")
logger.critical('Error: 无法发送邮件')
except smtplib.SMTPException as error:
logger.critical('Error: 无法发送邮件 {}'.format(error))
try:
if os.system(elementcmd):
logger.info('element 信息发送成功')
else:
os.system("kmg imu zengshuai 'haweye element Error'")
loger.critical('Error: element信息发送失败')
def load_applications():
"""
Load all of the applications server side code.
This is done by importing all of the packages inside
the applications directory.
Note: code which is registered using init.register()
isn't executed yet.
"""
path = os.path.dirname(__file__)
for module in os.listdir(path):
if module.startswith('_') or not os.path.isdir(os.sep.join([path, module])):
continue
try:
importlib.import_module('.' + module, 'parasite.applications')
except:
logger.critical(module + ' server side failed to initialize with the exception:')
logger.critical(traceback.format_exc())
def target_directory(self, target_mode):
target_directory = None
if target_mode == TARGET_MODE_FOLDER:
target_directory = self.target
elif target_mode == TARGET_MODE_FILE:
target_directory = self.target
return target_directory
else:
logger.critical(
'[PARSE-ARGS] exception target mode ({mode})'.format(
mode=target_mode))
exit()
logger.debug('[PARSE-ARGS] target directory: {directory}'.format(
directory=target_directory))
target_directory = os.path.abspath(target_directory)
if target_directory[-1] == '/':
return target_directory
else:
return u'{t}/'.format(t=target_directory)
def target_mode(self):
"""
Parse target mode (git/file/folder/compress)
:return: str
"""
target_mode = None
target_git_cases = ['http://', 'https://', 'ssh://']
for tgc in target_git_cases:
if self.target[0:len(tgc)] == tgc:
target_mode = TARGET_MODE_GIT
if os.path.isfile(self.target):
target_mode = TARGET_MODE_FILE
if os.path.isdir(self.target):
target_mode = TARGET_MODE_FOLDER
if target_mode is None:
logger.critical('[PARSE-ARGS] [-t <target>] can\'t empty!')
exit()
logger.debug(
'[PARSE-ARGS] Target Mode: {mode}'.format(mode=target_mode))
return target_mode
def target_directory(self, target_mode):
target_directory = None
if target_mode == TARGET_MODE_FOLDER:
target_directory = self.target
elif target_mode == TARGET_MODE_FILE:
target_directory = self.target
# 检查目标是否为zip
if os.path.splitext(target_directory)[-1] == '.zip':
try:
logger.info("[CLI] Target {} is zip, try to unzip.".format(
target_directory))
target_directory = un_zip(target_directory)
except zipfile.BadZipFile:
logger.warning(
"[CLI] file {} not zip".format(target_directory))
except OSError:
logger.warning(
"[CLI] file {} unzip error".format(target_directory))
return target_directory
else:
logger.critical(
'[PARSE-ARGS] exception target mode ({mode})'.format(
mode=target_mode))
exit()
logger.debug('[PARSE-ARGS] target directory: {directory}'.format(
directory=target_directory))
target_directory = os.path.abspath(target_directory)
if target_directory[-1] == '/':
return target_directory
else:
return u'{t}/'.format(t=target_directory)
def set_trace(self, *args, **kwargs):
"""Start a pdb debugger available via telnet, and optionally email people the endpoint
The endpoint will always be seen in the py.test runner output.
Keyword Args:
recipients: A list where, if set, an email will be sent to email addresses
in this list.
subject: If set, an optional custom email subject
"""
host, port = self.sock.getsockname()
endpoint = 'host {} port {}'.format(store.my_ip_address, port)
recipients = kwargs.pop('recipients', None)
if recipients:
# write and send an email
subject = kwargs.pop('subject', 'RDB Breakpoint: Manually invoked')
body = dedent("""\
A py.test run encountered an error. The remote debugger is running
on {} (TCP), waiting for telnet connection.
""").format(endpoint)
try:
smtp_server = smtp_conf['server']
smtp = smtplib.SMTP(smtp_server)
msg = MIMEText(body)
msg['Subject'] = subject
msg['To'] = ', '.join(recipients)
smtp.sendmail('*****@*****.**', recipients,
msg.as_string())
except socket.error:
logger.critical("Couldn't send email")
msg = 'Remote debugger listening on {}'.format(endpoint)
logger.critical(msg)
write_line(msg, red=True, bold=True)
self.sock.listen(1)
(client_socket, address) = self.sock.accept()
client_fh = client_socket.makefile('rw')
Pdb.__init__(self,
completekey='tab',
stdin=client_fh,
stdout=client_fh)
sys.stdout = sys.stdin = client_fh
Pdb.set_trace(self, *args, **kwargs)
msg = 'Debugger on {} shut down'.format(endpoint)
logger.critical(msg)
write_line(msg, green=True, bold=True)
def set_trace(self, *args, **kwargs):
"""Start a pdb debugger available via telnet, and optionally email people the endpoint
The endpoint will always be seen in the py.test runner output.
Keyword Args:
recipients: A list where, if set, an email will be sent to email addresses
in this list.
subject: If set, an optional custom email subject
"""
host, port = self.sock.getsockname()
endpoint = 'host {} port {}'.format(store.my_ip_address, port)
recipients = kwargs.pop('recipients', None)
if recipients:
# write and send an email
subject = kwargs.pop('subject', 'RDB Breakpoint: Manually invoked')
body = dedent("""\
A py.test run encountered an error. The remote debugger is running
on {} (TCP), waiting for telnet connection.
""").format(endpoint)
try:
smtp_server = smtp_conf['server']
smtp = smtplib.SMTP(smtp_server)
msg = MIMEText(body)
msg['Subject'] = subject
msg['To'] = ', '.join(recipients)
smtp.sendmail('*****@*****.**', recipients, msg.as_string())
except socket.error:
logger.critical("Couldn't send email")
msg = 'Remote debugger listening on {}'.format(endpoint)
logger.critical(msg)
write_line(msg, red=True, bold=True)
self.sock.listen(1)
(client_socket, address) = self.sock.accept()
client_fh = client_socket.makefile('rw')
Pdb.__init__(self, completekey='tab', stdin=client_fh, stdout=client_fh)
sys.stdout = sys.stdin = client_fh
Pdb.set_trace(self, *args, **kwargs)
msg = 'Debugger on {} shut down'.format(endpoint)
logger.critical(msg)
write_line(msg, green=True, bold=True)
def search(query, page, g, github_username):
mail_notice_list = []
webhook_notice_list = []
logger.info('开始抓取: tag is {} keyword is {}, page is {}'.format(
query.get('tag'), query.get('keyword'), page + 1))
try:
repos = g.search_code(query=query.get('keyword'),
sort="indexed",
order="desc")
github_col.update_one({'username': github_username}, {
'$set': {
'rate_remaining': int(g.get_rate_limit().search.remaining)
}
})
except Exception as error:
logger.critical(error)
logger.critical("触发限制啦")
return
try:
for repo in repos.get_page(page):
setting_col.update_one({'key': 'task'}, {
'$set': {
'key': 'task',
'pid': os.getpid(),
'last': timestamp()
}
},
upsert=True)
if not result_col.count({'_id': repo.sha}):
try:
code = str(repo.content).replace('\n', '')
except:
code = ''
leakage = {
'link': repo.html_url,
'project': repo.repository.full_name,
'project_url': repo.repository.html_url,
'_id': repo.sha,
'language': repo.repository.language,
'username': repo.repository.owner.login,
'avatar_url': repo.repository.owner.avatar_url,
'filepath': repo.path,
'filename': repo.name,
'security': 0,
'ignore': 0,
'tag': query.get('tag'),
'code': code,
}
try:
leakage['affect'] = get_affect_assets(repo.decoded_content)
except Exception as error:
logger.critical('{} {}'.format(error, leakage.get('link')))
leakage['affect'] = []
if int(repo.raw_headers.get('x-ratelimit-remaining')) == 0:
logger.critical('剩余使用次数: {}'.format(
repo.raw_headers.get('x-ratelimit-remaining')))
return
last_modified = datetime.datetime.strptime(
repo.last_modified, '%a, %d %b %Y %H:%M:%S %Z')
leakage['datetime'] = last_modified
leakage['timestamp'] = last_modified.timestamp()
in_blacklist = False
for blacklist in blacklist_col.find({}):
if blacklist.get('text').lower() in leakage.get(
'link').lower():
logger.warning('{} 包含白名单中的 {}'.format(
leakage.get('link'), blacklist.get('text')))
in_blacklist = True
if in_blacklist:
continue
if result_col.count({
"project": leakage.get('project'),
"ignore": 1
}):
continue
if not result_col.count({
"project": leakage.get('project'),
"filepath": leakage.get("filepath"),
"security": 0
}):
mail_notice_list.append(
'上传时间:{} 地址: <a href={}>{}/{}</a>'.format(
leakage.get('datetime'), leakage.get('link'),
leakage.get('project'), leakage.get('filename')))
webhook_notice_list.append('[{}/{}]({}) 上传于 {}'.format(
leakage.get('project').split('.')[-1],
leakage.get('filename'), leakage.get('link'),
leakage.get('datetime')))
try:
result_col.insert_one(leakage)
logger.info(leakage.get('project'))
except errors.DuplicateKeyError:
logger.info('已存在')
logger.info('抓取关键字：{} {}'.format(query.get('tag'),
leakage.get('link')))
except Exception as error:
if 'Not Found' not in error.data:
g, github_username = new_github()
search.schedule(args=(query, page, g, github_username),
delay=huey.pending_count() +
huey.scheduled_count())
logger.critical(error)
logger.error('抓取: tag is {} keyword is {}, page is {} 失败'.format(
query.get('tag'), query.get('keyword'), page + 1))
return
logger.info('抓取: tag is {} keyword is {}, page is {} 成功'.format(
query.get('tag'), query.get('keyword'), page + 1))
query_col.update_one({'tag': query.get('tag')}, {
'$set': {
'last': int(time.time()),
'status': 1,
'reason': '抓取第{}页成功'.format(page),
'api_total': repos.totalCount,
'found_total': result_col.count({'tag': query.get('tag')})
}
})
if setting_col.count({
'key': 'mail',
'enabled': True
}) and len(mail_notice_list):
main_content = '<h2>规则名称: {}</h2><br>{}'.format(
query.get('tag'), '<br>'.join(mail_notice_list))
send_mail(main_content)
logger.info(len(webhook_notice_list))
webhook_notice(query.get('tag'), webhook_notice_list)
def __init__(self,
target,
formatter,
output,
special_rules=None,
language=None,
black_path=None,
a_sid=None):
self.target = target
self.formatter = formatter
self.output = output if output else ""
if special_rules != None and special_rules != '':
self.special_rules = []
extension = '.py'
start_name = 'CVI_'
if ',' in special_rules:
# check rule name
s_rules = special_rules.split(',')
for sr in s_rules:
if extension not in sr:
sr += extension
if start_name not in sr:
sr = start_name + sr
if self._check_rule_name(sr):
self.special_rules.append(sr)
else:
logger.critical(
'[PARSE-ARGS] Rule {sr} not exist'.format(sr=sr))
else:
special_rules = start_name + special_rules + extension
if self._check_rule_name(special_rules):
self.special_rules = [special_rules]
else:
logger.critical(
'[PARSE-ARGS] Exception special rule name(e.g: CVI-110001): {sr}'
.format(sr=special_rules))
else:
self.special_rules = None
# check black pth list
if black_path != None and black_path != "":
self.black_path_list = []
if ',' in black_path:
self.black_path_list = [
x.strip() for x in black_path.split(',') if x != ""
]
logger.info("[INIT][PARSE_ARGS] Black Path list is {}".format(
self.black_path_list))
else:
self.black_path_list = []
logger.warning("[INIT][PARSE_ARGS] Black Path parse error.")
else:
self.black_path_list = []
# check and deal language
if language != None and language != "":
self.language = []
if ',' in language:
self.language = [
x.strip() for x in language.split(',') if x != ""
]
logger.info("[INIT][PARSE_ARGS] Language is {}".format(
self.language))
else:
self.language = [language.strip()]
logger.info("[INIT][PARSE_ARGS] Only one Language {}.".format(
self.language))
self.sid = a_sid
def scan(target_directory, a_sid=None, s_sid=None, special_rules=None, language=None, framework=None, file_count=0,
extension_count=0, files=None, tamper_name=None, is_unconfirm=False):
r = Rule(language)
vulnerabilities = r.vulnerabilities
rules = r.rules(special_rules)
find_vulnerabilities = []
newcore_function_list = {}
def store(result):
if result is not None and isinstance(result, list) is True:
for res in result:
res.file_path = res.file_path
find_vulnerabilities.append(res)
else:
logger.debug('[SCAN] [STORE] Not found vulnerabilities on this rule!')
async def start_scan(target_directory, rule, files, language, tamper_name):
result = scan_single(target_directory, rule, files, language, tamper_name, is_unconfirm, newcore_function_list)
store(result)
if len(rules) == 0:
logger.critical('no rules!')
return False
logger.info('[PUSH] {rc} Rules'.format(rc=len(rules)))
push_rules = []
scan_list = []
for idx, single_rule in enumerate(sorted(rules.keys())):
# init rule class
r = getattr(rules[single_rule], single_rule)
rule = r()
if rule.status is False and len(rules) != 1:
logger.info('[CVI_{cvi}] [STATUS] OFF, CONTINUE...'.format(cvi=rule.svid))
continue
# SR(Single Rule)
logger.debug("""[PUSH] [CVI_{cvi}] {idx}.{vulnerability}({language})""".format(
cvi=rule.svid,
idx=idx,
vulnerability=rule.vulnerability,
language=rule.language
))
# result = scan_single(target_directory, rule, files, language, tamper_name)
scan_list.append(start_scan(target_directory, rule, files, language, tamper_name))
# store(result)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(*scan_list))
loop.stop()
# print
data = []
data2 = []
table = PrettyTable(
['#', 'CVI', 'Rule(ID/Name)', 'Lang/CVE-id', 'Target-File:Line-Number',
'Commit(Author)', 'Source Code Content', 'Analysis'])
table.align = 'l'
trigger_rules = []
for idx, x in enumerate(find_vulnerabilities):
trigger = '{fp}:{ln}'.format(fp=x.file_path, ln=x.line_number)
commit = u'@{author}'.format(author=x.commit_author)
try:
code_content = x.code_content[:50].strip()
except AttributeError as e:
code_content = x.code_content.decode('utf-8')[:100].strip()
row = [idx + 1, x.id, x.rule_name, x.language, trigger, commit,
code_content.replace('\r\n', ' ').replace('\n', ' '), x.analysis]
row2 = [idx + 1, x.chain]
is_unconfirm_result = False
if "unconfirmed" in x.analysis.lower():
is_unconfirm_result = True
# save to database
sr = ScanResultTask(scan_task_id=a_sid, result_id=idx + 1, cvi_id=x.id, language=x.language,
vulfile_path=trigger, source_code=code_content.replace('\r\n', ' ').replace('\n', ' '),
result_type=x.analysis, is_unconfirm=is_unconfirm_result)
sr.save()
for chain in x.chain:
if type(chain) == tuple:
ResultFlow = get_resultflow_class(int(a_sid))
rf = ResultFlow(vul_id=idx + 1, node_type=chain[0], node_content=chain[1],
node_path=chain[2], node_lineno=chain[3])
rf.save()
data.append(row)
data2.append(row2)
table.add_row(row)
if x.id not in trigger_rules:
logger.debug(' > trigger rule (CVI-{cvi})'.format(cvi=x.id))
trigger_rules.append(x.id)
# clear
x.chain = ""
diff_rules = list(set(push_rules) - set(trigger_rules))
vn = len(find_vulnerabilities)
if vn == 0:
logger.info('[SCAN] Not found vulnerability!')
else:
logger.info("[SCAN] Trigger Rules: {tr} Vulnerabilities ({vn})\r\n{table}".format(tr=len(trigger_rules),
vn=len(find_vulnerabilities),
table=table))
# 输出chain for all
logger.info("[SCAN] Vulnerabilities Chain list: ")
for d in data2:
logger.info("[SCAN] Vul {}".format(d[0]))
for c in d[1]:
logger.info("[Chain] {}".format(c))
if type(c) is not tuple:
continue
show_context(c[2], c[3])
logger.info("[SCAN] ending\r\n" + '-' * (shutil.get_terminal_size().columns - 16))
if len(diff_rules) > 0:
logger.info(
'[SCAN] Not Trigger Rules ({l}): {r}'.format(l=len(diff_rules), r=','.join(diff_rules)))
# show detail about newcore function list
table2 = PrettyTable(
['#', 'NewFunction', 'OriginFunction', 'Related Rules id'])
table2.align = 'l'
idy = 0
for new_function_name in newcore_function_list:
# add new evil func in database
for svid in newcore_function_list[new_function_name]["svid"]:
if new_function_name:
nf = NewEvilFunc(svid=svid, scan_task_id=get_scan_id(), func_name=new_function_name,
origin_func_name=newcore_function_list[new_function_name]["origin_func_name"])
nf.save()
table2.add_row([idy + 1, new_function_name, newcore_function_list[new_function_name]["origin_func_name"], newcore_function_list[new_function_name]["svid"]])
idy += 1
if len(newcore_function_list) > 0:
logger.info("[SCAN] New evil Function list by NewCore:\r\n{}".format(table2))
# completed running data
if s_sid is not None:
Running(s_sid).data({
'code': 1001,
'msg': 'scan finished',
'result': {
'vulnerabilities': [x.__dict__ for x in find_vulnerabilities],
'language': ",".join(language),
'framework': framework,
'extension': extension_count,
'file': file_count,
'push_rules': len(rules),
'trigger_rules': len(trigger_rules),
'target_directory': target_directory
}
})
return True
def crawl(query, page):
mail_notice_list = []
webhook_notice_list = []
search_url = 'https://search.gitee.com/?skin=rec&type=code&q={1}&sort=last_indexed' \
'&pageno={0}'
session = gitee_login()
logger.info('Gitee开始抓取: tag is {} keyword is {}, page is {}'.format(
query.get('tag'), query.get('keyword'), page + 1))
totalCount = 0
for page in range(page + 1, page + 2):
try:
logger.info("Gitee ------ 启动抓取: {}".format(
search_url.format(page, query.get('keyword'))))
resp = session.get(search_url.format(page, query.get('keyword')))
logger.info("Gitee 启动抓取: {}".format(
search_url.format(page, query.get('keyword'))))
tree = etree.HTML(resp.text)
nodes = tree.xpath('//*[@id="hits-list"]/div[@class="item"]')
for node in nodes:
logger.info("Gitee 开始抓取节点")
totalCount += 1
# i = nodes.index(node) + 1
leakage = {}
leakage['affect'] = []
datetime_ = node.xpath(Gitee.DATETIME)[0].text
# print(datetime)
datetime_match = re.match("[^\d]*(?P<Date>\d+.*)", datetime_)
if not datetime_match:
leakage['datetime'] = _format_time(
datetime.datetime.now().date())
else:
leakage['datetime'] = _format_time(
datetime_match.groups("Date")[0])
leakage['timestamp'] = leakage.get('datetime').timestamp()
leakage['link'] = cut_tail(
node.xpath(Gitee.LINK)[0].attrib['href'])
leakage['filepath'] = node.xpath(Gitee.LINK)[0].text
leakage['filename'] = leakage.get("filepath").split("/")[-1]
# leakage['link'] = 'https://gitee.com' + realative_link
leakage['_id'] = _md5(leakage['link'])
logger.info("Gitee ****** 开始抓取节点 {}".format(
leakage['datetime']))
project_username = node.xpath(Gitee.USERNAME)[0].text
leakage["vendor"] = "GITEE"
leakage['username'] = project_username.split("/")[0]
leakage['project'] = project_username
leakage['project_url'] = cut_tail(
node.xpath(Gitee.USERNAME)[0].attrib['href'])
logger.info("Gitee 抓取到 {}".format(leakage.get("project_url")))
if result_col.find_one({"link": leakage['link'], "datetime": leakage['datetime']}) or \
result_col.find_one({'_id': leakage['_id']}):
continue
in_blacklist = False
for blacklist in blacklist_col.find({}):
if blacklist.get('text').lower() in leakage.get(
'link').lower():
logger.warning('{} 包含白名单中的 {}'.format(
leakage.get('link'), blacklist.get('text')))
in_blacklist = True
if in_blacklist:
continue
if result_col.count({
"project": leakage.get('project'),
"ignore": 1
}):
continue
#gitee中可以只有项目，没有代码
leakage['avatar_url'] = 'https://gitee.com/logo-black.svg'
raw_code = gitee_raw_code(leakage['link'])
leakage['code'] = base64.b64encode(
raw_code.encode("utf-8")).decode("utf-8")
try:
leakage['affect'] = get_affect_assets(raw_code)
except Exception as error:
logger.critical('{} {}'.format(error, leakage.get('link')))
leakage['affect'] = []
leakage['tag'] = query['tag']
# leakage['detail'] = etree.tostring(node,encoding='unicode').replace('{{', '<<').\
# replace('}}', '>>')
language_node = node.xpath(Gitee.LANGUAGE)
if language_node:
leakage['language'] = language_node[0].text.strip()
else:
leakage['language'] = 'Unknow'
leakage['security'] = 0
leakage['ignore'] = 0
if not result_col.count({
"project": leakage.get('project'),
"filepath": leakage.get("filepath"),
"security": 0
}):
mail_notice_list.append(
'上传时间:{} 地址: <a href={}>{}/{}</a>'.format(
leakage.get('datetime'), leakage.get('link'),
leakage.get('project'), leakage.get('filename')))
webhook_notice_list.append('[{}/{}]({}) 上传于 {}'.format(
leakage.get('project').split('/')[-1],
leakage.get('filename'), leakage.get('link'),
leakage.get('datetime')))
result_col.insert_one(leakage)
logger.info("Gitee 抓取到的结果: {}".format(
leakage.get("project_url")))
except Exception as e:
raise (e)
print(e)
logger.error("Gitee error is {}".format(e))
return
logger.info('Gitee抓取: tag is {} keyword is {}, page is {} 成功'.format(
query.get('tag'), query.get('keyword'), page + 1))
query_col.update_one({'tag': query.get('tag')}, {
'$set': {
'last': int(time.time()),
'status': 1,
'reason': '抓取第{}页成功'.format(page),
'api_total': totalCount,
'found_total': result_col.count({'tag': query.get('tag')})
}
})
if setting_col.count({
'key': 'mail',
'enabled': True
}) and len(mail_notice_list):
main_content = '<h2>规则名称: {}</h2><br>{}'.format(
query.get('tag'), '<br>'.join(mail_notice_list))
send_mail(main_content)
logger.info(len(webhook_notice_list))
webhook_notice(query.get('tag'), webhook_notice_list)
