def ScriptVulnInfo(script_id): page_number = int(request.args.get('p')) page_result = {} result = [] lens = Vuln.objects(script=script_id).count() page_result['total_page'] = str(lens // page_item_number + 1) if page_number > lens // page_item_number + 1: return json.dumps(page_result) if page_number * page_item_number < lens: vulns = Vuln.objects( script=script_id)[lens - (page_number * page_item_number):lens - (page_item_number * (page_number - 1))] else: vulns = Vuln.objects(script=script_id)[0:lens - (page_item_number * (page_number - 1))] for each in vulns: info = {} script = Script.objects(script_id=each.script).first() try: info['task_name'] = cgi.escape( Task.objects(task_id=each.task_id).first().task_name) except: info['task_name'] = u'该任务已删除,无法查看' info['task_id'] = each.task_id info['target'] = each.target info['script_name'] = script.script_name info['script_id'] = each.script info['message'] = each.message info['script_type'] = each.script_type info['time'] = str(each.create_time).split('.')[0] info['level'] = script.script_level result.append(info) page_result['info'] = result return json.dumps(page_result)
def TaskInfo(task_id): result = {} result['info'] = {} status = Status.objects(task_id=task_id).first() if not status: result['success'] = False result['message'] = u'任务ID不存在。' return json.dumps(result) if status.status == STATUS.WAIT: result['success'] = False result['message'] = u'任务尚未开始,请等待。' return json.dumps(result) if status.status == STATUS.RUN: result['success'] = False result['message'] = u'扫描正在进行中,请等待。' return json.dumps(result) if status.status == STATUS.CLOSE: result['success'] = False result['message'] = u'该任务已关闭,无法查看。' return json.dumps(result) if status.status == STATUS.FAIL: result['success'] = False result['message'] = str(status.warning) return json.dumps(result) task = Task.objects(task_id=task_id).first() if not task: result['success'] = False result['message'] = u'任务ID不存在.' return json.dumps(result) scan_result = Result.objects(task_id=task_id).first() result['info']['task_name'] = cgi.escape(task.task_name) result['info']['input_target'] = task.short_target result['info']['scan_mode'] = task.scan_mode result['info']['target_type'] = task.target_type result['info']['script_type'] = task.script_type result['info']['start_time'] = str(task.start_time).split('.')[0] result['info']['finish_time'] = str(task.finish_time).split('.')[0] result['info']['target_number'] = task.target_number result['info']['full_target'] = task.full_target result['info']['script_number'] = len(json.loads(task.script_info)) result['info']['high_count'] = scan_result.high_count result['info']['medium_count'] = scan_result.medium_count result['info']['low_count'] = scan_result.low_count result['info']['result'] = scan_result.result scripts_list = [] for script_name in json.loads(task.script_info): script_data = {} script = Script.objects(script_name=script_name).first() script_data['script_name'] = script_name script_data['script_title'] = script.script_title script_data['script_id'] = script.script_id script_data['script_level'] = script.script_level scripts_list.append(script_data) result['info']['scripts_info'] = scripts_list result['info']['warning'] = status.warning result['success'] = True result['message'] = '' return json.dumps(result)
def GetScripts(): result = [] scripts = Script.objects().all() for script in scripts: info = {} info['name'] = script.script_name info['id'] = script.script_id result.append(info) return json.dumps(result)
def ScriptList(show_type): result = [] page_result = {} if show_type == 'index': data = Vuln.objects.item_frequencies('script', normalize=True) data = OrderedDict( sorted(data.items(), key=lambda x: x[1], reverse=True)) for script_id, percent in data.items(): script = Script.objects(script_id=script_id).first() info = {} info['name'] = script.script_name info['create_time'] = script.script_update_time info['author'] = script.script_author info['title'] = script.script_title info['level'] = script.script_level result.append(info) if len(result) > 10: result = result[0:10] elif show_type == 'all': data = Script.objects.all() for each in data: info = {} info['id'] = each.script_id info['detail'] = each.script_info info['create_time'] = each.script_update_time info['author'] = each.script_author info['level'] = each.level result.append(info) result.reverse() elif show_type == 'page': page_number = int(request.args.get('p')) lens = len(Script.objects) page_result['total_page'] = str(lens // page_item_number + 1) if page_number > lens // page_item_number + 1: return json.dumps(page_result) if page_number * page_item_number < lens: data = Script.objects[lens - (page_number * page_item_number):lens - (page_item_number * (page_number - 1))] else: data = Script.objects[0:lens - (page_item_number * (page_number - 1))] for each in data: info = {} info['id'] = each.script_id info['detail'] = each.script_info info['create_time'] = each.script_update_time info['author'] = each.script_author info['level'] = each.script_level info['count'] = Vuln.objects(script=each.script_id).count() info['title'] = each.script_title result.append(info) result.reverse() page_result['info'] = result return json.dumps(page_result) return json.dumps(result)
def ScriptUpdate(): message = {} try: scripts_info = ScriptsInfo() Script.objects().delete() for script in scripts_info: if Script.objects(script_id=hashlib.md5( script['name']).hexdigest()).count() == 0: Script(script_id=hashlib.md5(script['name']).hexdigest(), script_name=script['name'], script_info=script['info'], script_author=script['author'], script_update_time=script['time'], script_level=script['level'], script_title=script['title']).save() except Exception, e: message['success'] = False message['message'] = repr(e) return json.dumps(message)
def get_company_info(tag, count): try: if tag['href'] == '': return soup = utilities.get_soup(tag['href']) code = soup.find('input', {'id': 'ap_sc_id'})['value'].lower() company = Script.objects(code=code).update_one(set__url=tag['href'], upsert=True) print(count, '.\t' + code) except Exception as e: print(type(e).__name__) print(e) print(count, '.\t' + tag) return
def partial_update(): connect('stock_exchange') codes = Trade.objects().distinct(field='code') watchlists = User.objects().only('watchlist') for watchlist in watchlists: codes.extend(watchlist.watchlist) codes = set(codes) # Testing Only # for i in Script.objects(code__in=codes): # get_company_data(i, count()) executor = futures.ThreadPoolExecutor() indResults = executor.map(update_index, Indicies.objects, count()) results = executor.map(get_company_data, Script.objects(code__in=codes), count())
def ScriptUpdate(): message = {} try: scripts_info = ScriptsInfo() Script.objects().delete() for script in scripts_info: if Script.objects(script_id = hashlib.md5(script['name']).hexdigest()).count() == 0: Script( script_id=hashlib.md5(script['name']).hexdigest(), script_name=script['name'], script_info=script['info'], script_author=script['author'], script_update_time=script['time'], script_level=script['level'], script_title=script['title'] ).save() except: message['success'] = False return json.dumps(message) message['success'] = True message['message'] = time.strftime("%Y/%m.%d %H:%M:%S", time.localtime()) return json.dumps(message)
def ScriptIdToScriptName(scripts): md5_reg = r'^[a-z0-9]{32}' script_list = [] script_tmp = [] if ',' in scripts: script_list = scripts.split(',') else: if re.match(re.compile(md5_reg), scripts): script_list.append(scripts) for script_id in script_list: if re.match(re.compile(md5_reg), script_id): script = Script.objects(script_id=script_id).first() if script == None: return '' script_tmp.append(script.script_name) else: return '' scripts_name = ','.join(script_tmp) return scripts_name
company['shareholding'] = holdings except (AttributeError): pass standalone = get_standalone_results(company['code'].upper()) if len(standalone) > 0 : company['standalone'] = standalone consolidated = get_consolidated_results(company['code'].upper()) if len(consolidated) > 0: company['consolidated'] = consolidated if any(key in company for key in ['shareholding', 'standalone', 'consolidated']): company.save() print(index, '\t ' + company['name']) else: print(index, '\t ' + company['name'] + '\t failed') if __name__ == "__main__": connect('stock_exchange') j = 0 # for i in Script.objects(name__ne=None): # try: # get_details(i, j) # j = j+1 # except Exception as e: # print(e) executor = futures.ThreadPoolExecutor() results = executor.map(get_details, Script.objects(name__ne=None), count())
def DataSearch(search_type): keyword = request.args.get('keyword') page_number = int(request.args.get('p')) result = [] page_result = {} if search_type == 'script': lens = len( Script.objects(__raw__={ 'script_info': re.compile(keyword) }).all()) page_result['total_page'] = str(lens // page_item_number + 1) if page_number > lens // page_item_number + 1: return json.dumps(page_result) if page_number * page_item_number < lens: scripts = Script.objects( __raw__={'script_info': re.compile(keyword) })[lens - (page_number * page_item_number):lens - (page_item_number * (page_number - 1))] else: scripts = Script.objects( __raw__={'script_info': re.compile(keyword) })[0:lens - (page_item_number * (page_number - 1))] for each in scripts: info = {} info['id'] = each.script_id info['detail'] = each.script_info info['create_time'] = each.script_update_time info['author'] = each.script_author info['level'] = each.script_level info['count'] = Vuln.objects(script=each.script_id).count() info['title'] = each.script_title result.append(info) elif search_type == 'task': lens = len( Status.objects(__raw__={ 'task_name': re.compile(keyword) }).all()) page_result['total_page'] = str(lens // page_item_number + 1) if page_number > lens // page_item_number + 1: return json.dumps(page_result) if page_number * page_item_number < lens: tasks = Status.objects(__raw__={'task_name': re.compile( keyword)})[lens - (page_number * page_item_number):lens - (page_item_number * (page_number - 1))] else: tasks = Status.objects(__raw__={'task_name': re.compile( keyword)})[0:lens - (page_item_number * (page_number - 1))] for each in tasks: info = {} info['create_time'] = str(each.create_time).split('.')[0] info['task_name'] = cgi.escape(each.task_name) info['task_id'] = each.task_id info['status'] = each.status info['progress'] = each.progress result.append(info) elif search_type == 'vuln': lens = len(Vuln.objects(__raw__={'target': re.compile(keyword)}).all()) page_result['total_page'] = str(lens // page_item_number + 1) if page_number > lens // page_item_number + 1: return json.dumps(page_result) if page_number * page_item_number < lens: vulns = Vuln.objects(__raw__={'target': re.compile( keyword)})[lens - (page_number * page_item_number):lens - (page_item_number * (page_number - 1))] else: vulns = Vuln.objects( __raw__={'target': re.compile(keyword)})[0:lens - (page_item_number * (page_number - 1))] for each in vulns: info = {} script = Script.objects(script_id=each.script).first() try: info['task_name'] = cgi.escape( Task.objects(task_id=each.task_id).first().task_name) except: info['task_name'] = u'该任务已删除,无法查看' info['task_id'] = each.task_id info['target'] = each.target info['script_name'] = script.script_name info['script_id'] = each.script info['message'] = each.message info['script_type'] = each.script_type info['time'] = str(each.create_time).split('.')[0] info['level'] = script.script_level result.append(info) page_result['info'] = result return json.dumps(page_result)
def VulnList(show_type): result = [] page_result = {} if show_type == 'index': data = Vuln.objects.item_frequencies('script', normalize=True) data = OrderedDict( sorted(data.items(), key=lambda x: x[1], reverse=True)) for script_id, percent in data.items(): info = {} info['script'] = Script.objects( script_id=script_id).first().script_name info['count'] = str(Vuln.objects(script=script_id).count()) info['percent'] = "%.02f%%" % (percent * 100) result.append(info) if len(result) > 10: result = result[0:10] elif show_type == 'all': data = Vuln.objects().all() for each in data: info = {} script = Script.objects(script_id=each.script).first() try: info['task_name'] = cgi.escape( Task.objects(task_id=each.task_id).first().task_name) except: info['task_name'] = u'该任务已删除,无法查看' info['task_id'] = each.task_id info['target'] = each.target info['script_name'] = script.script_name info['script_id'] = each.script info['message'] = each.message info['script_type'] = each.script_type info['time'] = str(each.create_time).split('.')[0] info['level'] = script.script_level result.append(info) result.reverse() elif show_type == 'page': page_number = int(request.args.get('p')) lens = len(Vuln.objects) page_result['total_page'] = str(lens // page_item_number + 1) if page_number > lens // page_item_number + 1: return json.dumps(page_result) if page_number * page_item_number < lens: data = Vuln.objects[lens - (page_number * page_item_number):lens - (page_item_number * (page_number - 1))] else: data = Vuln.objects[0:lens - (page_item_number * (page_number - 1))] for each in data: info = {} script = Script.objects(script_id=each.script).first() try: info['task_name'] = cgi.escape( Task.objects(task_id=each.task_id).first().task_name) except: info['task_name'] = u'该任务已删除,无法查看' info['task_id'] = each.task_id info['target'] = each.target info['script_name'] = script.script_name info['script_id'] = each.script info['message'] = each.message info['script_type'] = each.script_type info['time'] = str(each.create_time).split('.')[0] info['level'] = script.script_level result.append(info) result.reverse() page_result['info'] = result return json.dumps(page_result) return json.dumps(result)
def complete_update(): connect('stock_exchange') executor = futures.ThreadPoolExecutor() indResults = executor.map(update_index, Indicies.objects, count()) results = executor.map(get_company_data, Script.objects(name__ne=None), count())