def ScriptVulnInfo(script_id):
page_number = int(request.args.get('p'))
page_result = {}
result = []
lens = Vuln.objects(script=script_id).count()
page_result['total_page'] = str(lens // page_item_number + 1)
if page_number > lens // page_item_number + 1:
return json.dumps(page_result)
if page_number * page_item_number < lens:
vulns = Vuln.objects(
script=script_id)[lens - (page_number * page_item_number):lens -
(page_item_number * (page_number - 1))]
else:
vulns = Vuln.objects(script=script_id)[0:lens - (page_item_number *
(page_number - 1))]
for each in vulns:
info = {}
script = Script.objects(script_id=each.script).first()
try:
info['task_name'] = cgi.escape(
Task.objects(task_id=each.task_id).first().task_name)
except:
info['task_name'] = u'该任务已删除，无法查看'
info['task_id'] = each.task_id
info['target'] = each.target
info['script_name'] = script.script_name
info['script_id'] = each.script
info['message'] = each.message
info['script_type'] = each.script_type
info['time'] = str(each.create_time).split('.')[0]
info['level'] = script.script_level
result.append(info)
page_result['info'] = result
return json.dumps(page_result)
def TaskInfo(task_id):
result = {}
result['info'] = {}
status = Status.objects(task_id=task_id).first()
if not status:
result['success'] = False
result['message'] = u'任务ID不存在。'
return json.dumps(result)
if status.status == STATUS.WAIT:
result['success'] = False
result['message'] = u'任务尚未开始，请等待。'
return json.dumps(result)
if status.status == STATUS.RUN:
result['success'] = False
result['message'] = u'扫描正在进行中，请等待。'
return json.dumps(result)
if status.status == STATUS.CLOSE:
result['success'] = False
result['message'] = u'该任务已关闭，无法查看。'
return json.dumps(result)
if status.status == STATUS.FAIL:
result['success'] = False
result['message'] = str(status.warning)
return json.dumps(result)
task = Task.objects(task_id=task_id).first()
if not task:
result['success'] = False
result['message'] = u'任务ID不存在.'
return json.dumps(result)
scan_result = Result.objects(task_id=task_id).first()
result['info']['task_name'] = cgi.escape(task.task_name)
result['info']['input_target'] = task.short_target
result['info']['scan_mode'] = task.scan_mode
result['info']['target_type'] = task.target_type
result['info']['script_type'] = task.script_type
result['info']['start_time'] = str(task.start_time).split('.')[0]
result['info']['finish_time'] = str(task.finish_time).split('.')[0]
result['info']['target_number'] = task.target_number
result['info']['full_target'] = task.full_target
result['info']['script_number'] = len(json.loads(task.script_info))
result['info']['high_count'] = scan_result.high_count
result['info']['medium_count'] = scan_result.medium_count
result['info']['low_count'] = scan_result.low_count
result['info']['result'] = scan_result.result
scripts_list = []
for script_name in json.loads(task.script_info):
script_data = {}
script = Script.objects(script_name=script_name).first()
script_data['script_name'] = script_name
script_data['script_title'] = script.script_title
script_data['script_id'] = script.script_id
script_data['script_level'] = script.script_level
scripts_list.append(script_data)
result['info']['scripts_info'] = scripts_list
result['info']['warning'] = status.warning
result['success'] = True
result['message'] = ''
return json.dumps(result)
def GetScripts():
result = []
scripts = Script.objects().all()
for script in scripts:
info = {}
info['name'] = script.script_name
info['id'] = script.script_id
result.append(info)
return json.dumps(result)
def ScriptList(show_type):
result = []
page_result = {}
if show_type == 'index':
data = Vuln.objects.item_frequencies('script', normalize=True)
data = OrderedDict(
sorted(data.items(), key=lambda x: x[1], reverse=True))
for script_id, percent in data.items():
script = Script.objects(script_id=script_id).first()
info = {}
info['name'] = script.script_name
info['create_time'] = script.script_update_time
info['author'] = script.script_author
info['title'] = script.script_title
info['level'] = script.script_level
result.append(info)
if len(result) > 10:
result = result[0:10]
elif show_type == 'all':
data = Script.objects.all()
for each in data:
info = {}
info['id'] = each.script_id
info['detail'] = each.script_info
info['create_time'] = each.script_update_time
info['author'] = each.script_author
info['level'] = each.level
result.append(info)
result.reverse()
elif show_type == 'page':
page_number = int(request.args.get('p'))
lens = len(Script.objects)
page_result['total_page'] = str(lens // page_item_number + 1)
if page_number > lens // page_item_number + 1:
return json.dumps(page_result)
if page_number * page_item_number < lens:
data = Script.objects[lens -
(page_number * page_item_number):lens -
(page_item_number * (page_number - 1))]
else:
data = Script.objects[0:lens - (page_item_number *
(page_number - 1))]
for each in data:
info = {}
info['id'] = each.script_id
info['detail'] = each.script_info
info['create_time'] = each.script_update_time
info['author'] = each.script_author
info['level'] = each.script_level
info['count'] = Vuln.objects(script=each.script_id).count()
info['title'] = each.script_title
result.append(info)
result.reverse()
page_result['info'] = result
return json.dumps(page_result)
return json.dumps(result)
def ScriptUpdate():
message = {}
try:
scripts_info = ScriptsInfo()
Script.objects().delete()
for script in scripts_info:
if Script.objects(script_id=hashlib.md5(
script['name']).hexdigest()).count() == 0:
Script(script_id=hashlib.md5(script['name']).hexdigest(),
script_name=script['name'],
script_info=script['info'],
script_author=script['author'],
script_update_time=script['time'],
script_level=script['level'],
script_title=script['title']).save()
except Exception, e:
message['success'] = False
message['message'] = repr(e)
return json.dumps(message)
def get_company_info(tag, count):
try:
if tag['href'] == '': return
soup = utilities.get_soup(tag['href'])
code = soup.find('input', {'id': 'ap_sc_id'})['value'].lower()
company = Script.objects(code=code).update_one(set__url=tag['href'],
upsert=True)
print(count, '.\t' + code)
except Exception as e:
print(type(e).__name__)
print(e)
print(count, '.\t' + tag)
return
def partial_update():
connect('stock_exchange')
codes = Trade.objects().distinct(field='code')
watchlists = User.objects().only('watchlist')
for watchlist in watchlists:
codes.extend(watchlist.watchlist)
codes = set(codes)
# Testing Only
# for i in Script.objects(code__in=codes):
# get_company_data(i, count())
executor = futures.ThreadPoolExecutor()
indResults = executor.map(update_index, Indicies.objects, count())
results = executor.map(get_company_data, Script.objects(code__in=codes), count())
def ScriptUpdate():
message = {}
try:
scripts_info = ScriptsInfo()
Script.objects().delete()
for script in scripts_info:
if Script.objects(script_id = hashlib.md5(script['name']).hexdigest()).count() == 0:
Script(
script_id=hashlib.md5(script['name']).hexdigest(),
script_name=script['name'],
script_info=script['info'],
script_author=script['author'],
script_update_time=script['time'],
script_level=script['level'],
script_title=script['title']
).save()
except:
message['success'] = False
return json.dumps(message)
message['success'] = True
message['message'] = time.strftime("%Y/%m.%d %H:%M:%S", time.localtime())
return json.dumps(message)
def ScriptIdToScriptName(scripts):
md5_reg = r'^[a-z0-9]{32}'
script_list = []
script_tmp = []
if ',' in scripts:
script_list = scripts.split(',')
else:
if re.match(re.compile(md5_reg), scripts):
script_list.append(scripts)
for script_id in script_list:
if re.match(re.compile(md5_reg), script_id):
script = Script.objects(script_id=script_id).first()
if script == None:
return ''
script_tmp.append(script.script_name)
else:
return ''
scripts_name = ','.join(script_tmp)
return scripts_name
company['shareholding'] = holdings
except (AttributeError):
pass
standalone = get_standalone_results(company['code'].upper())
if len(standalone) > 0 :
company['standalone'] = standalone
consolidated = get_consolidated_results(company['code'].upper())
if len(consolidated) > 0:
company['consolidated'] = consolidated
if any(key in company for key in ['shareholding', 'standalone', 'consolidated']):
company.save()
print(index, '\t ' + company['name'])
else:
print(index, '\t ' + company['name'] + '\t failed')
if __name__ == "__main__":
connect('stock_exchange')
j = 0
# for i in Script.objects(name__ne=None):
# try:
# get_details(i, j)
# j = j+1
# except Exception as e:
# print(e)
executor = futures.ThreadPoolExecutor()
results = executor.map(get_details, Script.objects(name__ne=None), count())
def DataSearch(search_type):
keyword = request.args.get('keyword')
page_number = int(request.args.get('p'))
result = []
page_result = {}
if search_type == 'script':
lens = len(
Script.objects(__raw__={
'script_info': re.compile(keyword)
}).all())
page_result['total_page'] = str(lens // page_item_number + 1)
if page_number > lens // page_item_number + 1:
return json.dumps(page_result)
if page_number * page_item_number < lens:
scripts = Script.objects(
__raw__={'script_info': re.compile(keyword)
})[lens - (page_number * page_item_number):lens -
(page_item_number * (page_number - 1))]
else:
scripts = Script.objects(
__raw__={'script_info': re.compile(keyword)
})[0:lens - (page_item_number * (page_number - 1))]
for each in scripts:
info = {}
info['id'] = each.script_id
info['detail'] = each.script_info
info['create_time'] = each.script_update_time
info['author'] = each.script_author
info['level'] = each.script_level
info['count'] = Vuln.objects(script=each.script_id).count()
info['title'] = each.script_title
result.append(info)
elif search_type == 'task':
lens = len(
Status.objects(__raw__={
'task_name': re.compile(keyword)
}).all())
page_result['total_page'] = str(lens // page_item_number + 1)
if page_number > lens // page_item_number + 1:
return json.dumps(page_result)
if page_number * page_item_number < lens:
tasks = Status.objects(__raw__={'task_name': re.compile(
keyword)})[lens - (page_number * page_item_number):lens -
(page_item_number * (page_number - 1))]
else:
tasks = Status.objects(__raw__={'task_name': re.compile(
keyword)})[0:lens - (page_item_number * (page_number - 1))]
for each in tasks:
info = {}
info['create_time'] = str(each.create_time).split('.')[0]
info['task_name'] = cgi.escape(each.task_name)
info['task_id'] = each.task_id
info['status'] = each.status
info['progress'] = each.progress
result.append(info)
elif search_type == 'vuln':
lens = len(Vuln.objects(__raw__={'target': re.compile(keyword)}).all())
page_result['total_page'] = str(lens // page_item_number + 1)
if page_number > lens // page_item_number + 1:
return json.dumps(page_result)
if page_number * page_item_number < lens:
vulns = Vuln.objects(__raw__={'target': re.compile(
keyword)})[lens - (page_number * page_item_number):lens -
(page_item_number * (page_number - 1))]
else:
vulns = Vuln.objects(
__raw__={'target': re.compile(keyword)})[0:lens -
(page_item_number *
(page_number - 1))]
for each in vulns:
info = {}
script = Script.objects(script_id=each.script).first()
try:
info['task_name'] = cgi.escape(
Task.objects(task_id=each.task_id).first().task_name)
except:
info['task_name'] = u'该任务已删除，无法查看'
info['task_id'] = each.task_id
info['target'] = each.target
info['script_name'] = script.script_name
info['script_id'] = each.script
info['message'] = each.message
info['script_type'] = each.script_type
info['time'] = str(each.create_time).split('.')[0]
info['level'] = script.script_level
result.append(info)
page_result['info'] = result
return json.dumps(page_result)
def VulnList(show_type):
result = []
page_result = {}
if show_type == 'index':
data = Vuln.objects.item_frequencies('script', normalize=True)
data = OrderedDict(
sorted(data.items(), key=lambda x: x[1], reverse=True))
for script_id, percent in data.items():
info = {}
info['script'] = Script.objects(
script_id=script_id).first().script_name
info['count'] = str(Vuln.objects(script=script_id).count())
info['percent'] = "%.02f%%" % (percent * 100)
result.append(info)
if len(result) > 10:
result = result[0:10]
elif show_type == 'all':
data = Vuln.objects().all()
for each in data:
info = {}
script = Script.objects(script_id=each.script).first()
try:
info['task_name'] = cgi.escape(
Task.objects(task_id=each.task_id).first().task_name)
except:
info['task_name'] = u'该任务已删除，无法查看'
info['task_id'] = each.task_id
info['target'] = each.target
info['script_name'] = script.script_name
info['script_id'] = each.script
info['message'] = each.message
info['script_type'] = each.script_type
info['time'] = str(each.create_time).split('.')[0]
info['level'] = script.script_level
result.append(info)
result.reverse()
elif show_type == 'page':
page_number = int(request.args.get('p'))
lens = len(Vuln.objects)
page_result['total_page'] = str(lens // page_item_number + 1)
if page_number > lens // page_item_number + 1:
return json.dumps(page_result)
if page_number * page_item_number < lens:
data = Vuln.objects[lens - (page_number * page_item_number):lens -
(page_item_number * (page_number - 1))]
else:
data = Vuln.objects[0:lens - (page_item_number *
(page_number - 1))]
for each in data:
info = {}
script = Script.objects(script_id=each.script).first()
try:
info['task_name'] = cgi.escape(
Task.objects(task_id=each.task_id).first().task_name)
except:
info['task_name'] = u'该任务已删除，无法查看'
info['task_id'] = each.task_id
info['target'] = each.target
info['script_name'] = script.script_name
info['script_id'] = each.script
info['message'] = each.message
info['script_type'] = each.script_type
info['time'] = str(each.create_time).split('.')[0]
info['level'] = script.script_level
result.append(info)
result.reverse()
page_result['info'] = result
return json.dumps(page_result)
return json.dumps(result)
def complete_update():
connect('stock_exchange')
executor = futures.ThreadPoolExecutor()
indResults = executor.map(update_index, Indicies.objects, count())
results = executor.map(get_company_data, Script.objects(name__ne=None), count())
