def task(self,req,threadname):
self.logger and self.logger.info('%s 端口扫描 执行任务中%s', threadname,str(datetime.datetime.now()))
# print req[0],req[1],req[2],req[3]
if req[3]!='open':
return ''
ip=req[1]
port=req[2]
productname=req[4]
nmapscript=req[5]
head=None
ans=None
hackinfo=''
keywords=''
if req[0]=='http' or req[0]=='https':
if ip[0:4]=='http':
address=ip+':'+port
else:
if port=='443':
address='https'+'://'+ip+':'+port
else:
address=req[0]+'://'+ip+':'+port
print address
head,ans = self.connectpool.getConnect(address)
from template_identify import page_identify
keywords,hackinfo=page_identify.identify_main(head=head,context=ans,ip=ip,port=port,productname=productname,protocol=req[0],nmapscript=nmapscript)
else:
head,ans,keywords,hackinfo=self.portscan.do_scan(head=head,context=ans,ip=ip,port=port,name=req[0],productname=productname,nmapscript=nmapscript)
# print ans
# self.sqlTool.connectdb()
localtime=str(time.strftime("%Y-%m-%d %X", time.localtime()))
insertdata=[]
temp=str(ans)
head=SQLTool.escapewordby(head)
msg=SQLTool.escapewordby(temp)
hackinfomsg=SQLTool.escapewordby(hackinfo)
keywords=SQLTool.escapewordby(keywords)
import Sqldata
insertdata.append((ip,port,localtime,str(head),msg,str(port),hackinfomsg,keywords))
extra=' on duplicate key update detail=\''+msg+'\' ,head=\''+str(head)+'\', timesearch=\''+localtime+'\',hackinfo=\''+hackinfomsg+'\',keywords=\''+str(keywords)+'\''
sqldatawprk=[]
dic={"table":self.config.porttable,"select_params":['ip','port','timesearch','detail','head','portnumber','hackinfo','keywords'],"insert_values":insertdata,"extra":extra}
tempwprk=Sqldata.SqlData('inserttableinfo_byparams',dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
# inserttableinfo_byparams(table=self.config.porttable,select_params=['ip','port','timesearch','detail'],insert_values=insertdata,extra=extra)
# self.sqlTool.closedb()
self.logger and self.logger.info('%s 端口扫描 任务结束%s', threadname,str(datetime.datetime.now()))
return ans
def task(self,req,threadname):
print 'the ip is '+req
ip=req
jsondata=webtool.getLocationinfo(req)
country,country_id,area,area_id,region,region_id,city,city_id,county,county_id,isp,isp_id=getlocationjsondata(jsondata)
localtime=str(time.strftime("%Y-%m-%d %X", time.localtime()))
insertdata=[]
insertdata.append((ip,country,country_id,area,area_id,region,region_id,city,city_id,county,county_id,isp,isp_id,localtime))
extra=' on duplicate key update updatetime='+SQLTool.formatstring(localtime)+',country='+SQLTool.formatstring(country)+', country_id='+SQLTool.formatstring(country_id)+',area='+SQLTool.formatstring(area)+', area_id='+SQLTool.formatstring(area_id)+',region='+SQLTool.formatstring(region)+', region_id='+SQLTool.formatstring(region_id)+',city='+SQLTool.formatstring(city)+', city_id='+SQLTool.formatstring(city_id)+',county='+SQLTool.formatstring(county)+', county_id='+SQLTool.formatstring(county_id)+',isp='+SQLTool.formatstring(isp)+', isp_id='+SQLTool.formatstring(isp_id)
sqldatawprk=[]
dic={"table":self.config.iptable,"select_params":['ip','country','country_id','area','area_id','region','region_id','city','city_id','county','county_id','isp','isp_id','updatetime'],"insert_values":insertdata,"extra":extra}
if self.islocalwork==0:
tempdata={"func":'inserttableinfo_byparams',"dic":dic}
jsondata=uploaditem.UploadData(url=self.webconfig.upload_ip_info,way='POST',params=tempdata)
sqldatawprk.append(jsondata)
self.uploadwork.add_work(sqldatawprk)
else:
tempwprk=Sqldata.SqlData('inserttableinfo_byparams',dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
time.sleep(0.1)
ans=''
return ans
def __init__(self,isThread=1): TaskTool.__init__(self,isThread) self.sqlhelp=SQLTool.getObject() self.sqlhelp.connectdb()
def __init__(self, isThread=1):
TaskTool.__init__(self, isThread)
self.sqlhelp = SQLTool.getObject()
self.sqlhelp.connectdb()
def task(self, req, threadname):
self.logger and self.logger.info('%s 端口扫描 执行任务中%s', threadname,
str(datetime.datetime.now()))
# print req[0],req[1],req[2],req[3]
if req[3] != 'open':
return ''
ip = req[1]
port = req[2]
productname = req[4]
nmapscript = req[5]
head = None
ans = None
hackinfo = ''
keywords = ''
if req[0] == 'http' or req[0] == 'https':
if ip[0:4] == 'http':
address = ip + ':' + port
else:
if port == '443':
address = 'https' + '://' + ip + ':' + port
else:
address = req[0] + '://' + ip + ':' + port
print address
head, ans = self.connectpool.getConnect(address)
from template_identify import page_identify
keywords, hackinfo = page_identify.identify_main(
head=head,
context=ans,
ip=ip,
port=port,
productname=productname,
protocol=req[0],
nmapscript=nmapscript)
else:
head, ans, keywords, hackinfo = self.portscan.do_scan(
head=head,
context=ans,
ip=ip,
port=port,
name=req[0],
productname=productname,
nmapscript=nmapscript)
# print ans
# self.sqlTool.connectdb()
localtime = str(time.strftime("%Y-%m-%d %X", time.localtime()))
insertdata = []
temp = str(ans)
head = SQLTool.escapewordby('{' + head + '}')
msg = SQLTool.escapewordby('{' + temp + '}')
hackinfomsg = SQLTool.escapewordby(hackinfo)
keywords = SQLTool.escapewordby(keywords)
import Sqldata
insertdata.append((ip, port, localtime, str(head), msg, str(port),
hackinfomsg, keywords))
extra = ' on duplicate key update detail=\'' + msg + '\' ,head=\'' + str(
head
) + '\', timesearch=\'' + localtime + '\',hackinfo=\'' + hackinfomsg + '\',keywords=\'' + str(
keywords) + '\''
sqldatawprk = []
dic = {
"table":
self.config.porttable,
"select_params": [
'ip', 'port', 'timesearch', 'detail', 'head', 'portnumber',
'hackinfo', 'keywords'
],
"insert_values":
insertdata,
"extra":
extra
}
tempwprk = Sqldata.SqlData('inserttableinfo_byparams', dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
# inserttableinfo_byparams(table=self.config.porttable,select_params=['ip','port','timesearch','detail'],insert_values=insertdata,extra=extra)
# self.sqlTool.closedb()
self.logger and self.logger.info('%s 端口扫描 任务结束%s', threadname,
str(datetime.datetime.now()))
return ans
def task(self, ip, threadname):
print 'getLocationTool::task() ip location do ......the ip is ' + ip
jsondata = webtool.getLocationinfo(
ip) #使用ip.taobao.com获取该ip对应的信息，返回一个json结构的字典
country, country_id, area, area_id, region, region_id, city, city_id, county, county_id, isp, isp_id = getlocationjsondata(
jsondata)
localtime = str(time.strftime("%Y-%m-%d %X", time.localtime()))
insertdata = []
insertdata.append(
(ip, country, country_id, area, area_id, region, region_id, city,
city_id, county, county_id, isp, isp_id, localtime))
extra = ' on duplicate key update updatetime=' + SQLTool.formatstring(
localtime) + ',country=' + SQLTool.formatstring(
country) + ', country_id=' + SQLTool.formatstring(
country_id) + ',area=' + SQLTool.formatstring(
area) + ', area_id=' + SQLTool.formatstring(
area_id) + ',region=' + SQLTool.formatstring(
region
) + ', region_id=' + SQLTool.formatstring(
region_id) + ',city=' + SQLTool.formatstring(
city
) + ', city_id=' + SQLTool.formatstring(
city_id
) + ',county=' + SQLTool.formatstring(
county
) + ', county_id=' + SQLTool.formatstring(
county_id
) + ',isp=' + SQLTool.formatstring(
isp) + ', isp_id=' + SQLTool.formatstring(
isp_id)
sqldatawork = []
dic = {
"table":
self.config.iptable,
"select_params": [
'ip', 'country', 'country_id', 'area', 'area_id', 'region',
'region_id', 'city', 'city_id', 'county', 'county_id', 'isp',
'isp_id', 'updatetime'
],
"insert_values":
insertdata,
"extra":
extra
}
# 和之前的手工写入数据库操作一样
tempwork = Sqldata.SqlData('inserttableinfo_byparams', dic)
sqldatawork.append(tempwork)
self.sqlTool.add_work(sqldatawork)
del insertdata
time.sleep(0.2)
ans = ''
return ans
def callback_result(self,scan_result):
tmp=scan_result
for i in tmp['scan'].keys():
host=i
result=''
try:
# result = u"ip地址:%s 主机名:%s ...... %s\n" %(host,tmp['scan'][host].get('hostnames','null'),tmp['scan'][host]['status'].get('state','null'))
# self.sqlTool.connectdb()
# print tmp['scan'][host].get('hostname','null')
# if 'osclass' in tmp['scan'][host].keys():
# result +=u"系统信息 ： %s %s %s 准确度:%s \n" % (str(tmp['scan'][host]['osclass'].get('vendor','null')),str(tmp['scan'][host]['osclass'].get('osfamily','null')),str(tmp['scan'][host]['osclass'].get('osgen','null')),str(tmp['scan'][host]['osclass'].get('accuracy','null')))
# print result
temphosts=str(host)
localtime=str(time.strftime("%Y-%m-%d %X", time.localtime()))
self.getlocationtool.add_work([temphosts])
try :
tempvendor='null'
temposfamily='null'
temposgen='null'
tempaccuracy='null'
if len(tmp['scan'][host]['osmatch'])>0 and len(tmp['scan'][host]['osmatch'][0]['osclass'])>0:
tempvendor=str(tmp['scan'][host]['osmatch'][0]['osclass'][0].get('vendor','null'))
temposfamily=str(tmp['scan'][host]['osmatch'][0]['osclass'][0].get('osfamily','null'))
temposgen=str(tmp['scan'][host]['osmatch'][0]['osclass'][0].get('osgen','null'))
tempaccuracy=str(tmp['scan'][host]['osmatch'][0]['osclass'][0].get('accuracy','null'))
temphostname=''
tempdecide=tmp['scan'][host].get('hostnames',[])
if len(tempdecide)>0:
for y in tmp['scan'][host]['hostnames']:
temphostname+=str(y.get('name','unknow'))+' '
tempstate=str(tmp['scan'][host]['status'].get('state','null'))
# print temphosts,tempvendor,temposfamily,temposgen,tempaccuracy,localtime
# self.sqlTool.replaceinserttableinfo_byparams(table=self.config.iptable,select_params= ['ip','vendor','osfamily','osgen','accurate','updatetime','hostname','state'],insert_values= [(temphosts,tempvendor,temposfamily,temposgen,tempaccuracy,localtime,temphostname,tempstate)])
sqldatawprk=[]
dic={"table":self.config.iptable,"select_params": ['ip','vendor','osfamily','osgen','accurate','updatetime','hostname','state'],"insert_values": [(temphosts,tempvendor,temposfamily,temposgen,tempaccuracy,localtime,temphostname,tempstate)]}
tempwprk=Sqldata.SqlData('replaceinserttableinfo_byparams',dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
except Exception,e:
print 'nmap system error d '+str(e)
if 'tcp' in tmp['scan'][host].keys():
ports = tmp['scan'][host]['tcp'].keys()
for port in ports:
# portinfo = " port : %s name:%s state : %s product : %s version :%s script:%s \n" %(port,tmp['scan'][host]['tcp'][port].get('name',''),tmp['scan'][host]['tcp'][port].get('state',''), tmp['scan'][host]['tcp'][port].get('product',''),tmp['scan'][host]['tcp'][port].get('version',''),tmp['scan'][host]['tcp'][port].get('script',''))
tempport=str(port)
tempportname=str(tmp['scan'][host]['tcp'][port].get('name',''))
tempportstate=str(tmp['scan'][host]['tcp'][port].get('state',''))
tempproduct=str(tmp['scan'][host]['tcp'][port].get('product',''))
tempportversion=str(tmp['scan'][host]['tcp'][port].get('version',''))
tempscript=SQLTool.decodestr(str(tmp['scan'][host]['tcp'][port].get('script',{})))
# self.sqlTool.replaceinserttableinfo_byparams(table=self.config.porttable,select_params= ['ip','port','timesearch','state','name','product','version','script'],insert_values= [(temphosts,tempport,localtime,tempportstate,tempportname,tempproduct,tempportversion,tempscript)])
sqldatawprk=[]
dic={"table":self.config.porttable,"select_params": ['ip','port','timesearch','state','name','product','version','script','portnumber'],"insert_values": [(temphosts,tempport,localtime,tempportstate,tempportname,tempproduct,tempportversion,tempscript,str(tempport))]}
tempwprk=Sqldata.SqlData('replaceinserttableinfo_byparams',dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
self.portscan.add_work([(tempportname,temphosts,tempport,tempportstate,tempproduct,tempscript)])
elif 'udp' in tmp['scan'][host].keys():
ports = tmp['scan'][host]['udp'].keys()
for port in ports:
# portinfo = " port : %s name:%s state : %s product : %s version :%s script:%s \n" %(port,tmp['scan'][host]['udp'][port].get('name',''),tmp['scan'][host]['udp'][port].get('state',''), tmp['scan'][host]['udp'][port].get('product',''),tmp['scan'][host]['udp'][port].get('version',''),tmp['scan'][host]['udp'][port].get('script',''))
# result = result + portinfo
tempport=str(port)
tempportname=str(tmp['scan'][host]['udp'][port].get('name',''))
tempportstate=str(tmp['scan'][host]['udp'][port].get('state',''))
tempproduct=str(tmp['scan'][host]['udp'][port].get('product',''))
tempportversion=str(tmp['scan'][host]['udp'][port].get('version',''))
tempscript=str(tmp['scan'][host]['udp'][port].get('script',''))
# self.sqlTool.replaceinserttableinfo_byparams(table=self.config.porttable,select_params= ['ip','port','timesearch','state','name','product','version','script'],insert_values= [(temphosts,tempport,localtime,tempportstate,tempportname,tempproduct,tempportversion,tempscript)])
sqldatawprk=[]
dic={"table":self.config.porttable,"select_params": ['ip','port','timesearch','state','name','product','version','script','portnumber'],"insert_values": [(temphosts,tempport,localtime,tempportstate,tempportname,tempproduct,tempportversion,tempscript,str(tempport))]}
tempwprk=Sqldata.SqlData('replaceinserttableinfo_byparams',dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
except Exception,e:
print 'nmap error'+str(e)
def task(self,req,threadname):
# print ("\n======================portscantask::task() req:%s======================\n"%str(req))
if req[3]!='open':
return ''
protocal = req[0]
ip = req[1]
port = req[2]
productname = req[4]
nmapscript = req[5]
head = None
page = None
hackresults = ''
keywords = ''
webkey = ''
webtitle = ''
self.logger.info(' 端口[%s:%s]扫描%s执行任务中%s', protocal, port, threadname, str(datetime.datetime.now()))
if port in ['3306', '873', '22', '21']:
# mysql/ftp/rsync/ssh四个检测，暴力破解尝试登录；head和page无返回，为空
head, page, keywords, hackresults = self.portscan.do_scan(head=head,context=page,ip=ip,port=port,name=protocal,productname=productname,nmapscript=nmapscript)
import webutil
webinfo = webutil.getwebinfo(page)
webkey = webinfo['keywords']
webtitle = webinfo['title']
self.logger.info('webutil.getwebinfo(%s:%s) method_2 \nkeywords:%s\ntitle:%s\n', ip, str(port), webkey, webtitle)
# 7001端口是Freak88, Weblogic默认端口
# if (protocal == 'http' or protocal == 'https') or (protocal in ['tcpwrapped', 'None'] and port in ['80','8080','7001']):
else:
if port == '443':
address = 'https' + '://' + ip + ':' + port
elif ip[0:4] == 'http':
address = ip + ':' + port
else:
address = 'http://' + ip + ':' + port
# if ip[0:4] == 'http':
# address = ip+':'+port
# else:
# if port == '443':
# address='https'+'://'+ip+':'+port
# else:
# if protocal == 'tcpwrapped' and port in ['80','8080','7001']:
# address = 'http://' + ip + ':' + port
# else: # None, 不合法?ftp/smtp...貌似无法访问，都是error
# address = protocal+'://'+ip+':'+port
# 获取网页反馈的头部和整个网页信息(urllib2, requests)
self.logger.info('get %s\'s head and context', address)
head, page = self.connectpool.getConnect(address)
import webutil
# 获取网页的关键词和网站标题
webinfo = webutil.getwebinfo(page)
webkey = webinfo['keywords']
webtitle = webinfo['title']
self.logger.info('webutil.getwebinfo(%s) method_1 \nkeywords:%s\ntitle:%s\n', address, webkey, webtitle)
try:
# 调用检测功能（http/poc/fuzz，目前只开源了fuzz检测）
# httpdect(headdect) 可以获得keywords和hackresults信息, 后续要探究下这部分怎么解析, 所以目前返回的结果为空
# pocsearch 后续也要加入
from detection import page_identify
keywords, hackresults = page_identify.identify_main(head=head,context=page,ip=ip,port=port,productname=productname,protocol=protocal,nmapscript=nmapscript)
except:
pass
# print page
# self.sqlTool.connectdb()
localtime=str(time.strftime("%Y-%m-%d %X", time.localtime()))
insertdata=[]
temp = str(page)
# 通过转义存入数据库，不然一些\'和sql语句冲突，无法存入！str(word).replace("'", "'")
# str(MySQLdb.escape_string(str(decodestr(word))))
head = SQLTool.escapewordby('{'+head+'}')
msg = SQLTool.escapewordby('{'+temp+'}')
hackresultsmsg = SQLTool.escapewordby(str(hackresults))
keywords = SQLTool.escapewordby(str(keywords))
import Sqldata
insertdata.append((ip,port,localtime,msg,str(head),str(port),hackresultsmsg,keywords,webkey,webtitle))
extra = ' on duplicate key update detail=\''+msg+'\' ,head=\''+str(head)+'\', timesearch=\''+localtime+'\', hackresults=\''+hackresultsmsg+'\',keywords=\''+str(keywords)+'\',webkeywords=\''+webkey+'\',webtitle=\''+webtitle+'\''
sqldatawprk = []
dic = {"table":self.config.porttable,"select_params":['ip','port','timesearch','detail','head','portnumber','hackresults','keywords','webkeywords','webtitle'],"insert_values":insertdata,"extra":extra}
tempwprk = Sqldata.SqlData('inserttableinfo_byparams',dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
# inserttableinfo_byparams(table=self.config.porttable,select_params=['ip','port','timesearch','detail'],insert_values=insertdata,extra=extra)
# self.sqlTool.closedb()
self.logger.info('%s 端口[%s]扫描任务结束%s', port, threadname,str(datetime.datetime.now()))
return page
def task(self, req, threadname):
print 'ip location do ......the ip is ' + req
ip = req
jsondata = webtool.getLocationinfo(req)
country, country_id, area, area_id, region, region_id, city, city_id, county, county_id, isp, isp_id = getlocationjsondata(
jsondata)
localtime = str(time.strftime("%Y-%m-%d %X", time.localtime()))
insertdata = []
insertdata.append(
(ip, country, country_id, area, area_id, region, region_id, city,
city_id, county, county_id, isp, isp_id, localtime))
extra = ' on duplicate key update updatetime=' + SQLTool.formatstring(
localtime) + ',country=' + SQLTool.formatstring(
country) + ', country_id=' + SQLTool.formatstring(
country_id) + ',area=' + SQLTool.formatstring(
area) + ', area_id=' + SQLTool.formatstring(
area_id) + ',region=' + SQLTool.formatstring(
region
) + ', region_id=' + SQLTool.formatstring(
region_id) + ',city=' + SQLTool.formatstring(
city
) + ', city_id=' + SQLTool.formatstring(
city_id
) + ',county=' + SQLTool.formatstring(
county
) + ', county_id=' + SQLTool.formatstring(
county_id
) + ',isp=' + SQLTool.formatstring(
isp) + ', isp_id=' + SQLTool.formatstring(
isp_id)
sqldatawprk = []
dic = {
"table":
self.config.iptable,
"select_params": [
'ip', 'country', 'country_id', 'area', 'area_id', 'region',
'region_id', 'city', 'city_id', 'county', 'county_id', 'isp',
'isp_id', 'updatetime'
],
"insert_values":
insertdata,
"extra":
extra
}
tempwprk = Sqldata.SqlData('inserttableinfo_byparams', dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
time.sleep(0.1)
ans = ''
return ans
def html2table(htmlData):
pattern = re.compile(r'<title>anhui \| CMS \|(.*?)</title>', re.S)
tt = re.findall(pattern, htmlData)[0].encode('utf-8').strip()
pattern = re.compile(
r'<table class="table table-striped table-bordered table-hover" id="datatable">.*?</table>',
re.S)
content = re.findall(pattern, htmlData)
pattern = re.compile(r'<th class="text-center">(.*?)</th>', re.S)
titleList = re.findall(pattern, content[0].encode('utf-8'))
newlist = []
for title in titleList:
s = title.replace('(', '').replace(')', '').replace('（', '').replace(
'）', '').replace('/', '')
title = s
newlist.append(s)
titleList = newlist
if titleList[0] == '':
titleList.pop(0)
if tt == '订单工时分类明细':
titleList[2] = '日期'
sql = SQLTool.Mysql()
sql.createTable(tt, titleList)
# reobj = re.compile(r'<!-- <td>.*?</td> -->',re.S)
result = re.sub(r'<!--.*?<td>.*?</td>.*?-->', '',
content[0].encode('utf-8'))
# print(titleList)
# print(result)
pattern = re.compile(r'<tr class="text-center">(.*?)</tr>', re.S)
tableContent = re.findall(pattern, result)
contentList = []
for oneList in tableContent:
pattern = re.compile(r'<td.*?>(.*?)</td>', re.S)
record = re.findall(pattern, oneList)
tp = record
for i, t in enumerate(tp):
if 'fa-search' in record[i]:
record.pop(i)
if '<a href="#" data-toggle="dropdown"' in record[i]:
pattern = re.compile(
r'(.*?)<a href=\"#\" data-toggle=\"dropdown\"', re.S)
r = re.findall(pattern, record[i])[0].replace('/n', '')
record[i] = r
contentList.append(record)
pattern = re.compile(
r'<option value=.*? selected="selected">(.*?)</option>', re.S)
name = re.findall(pattern, htmlData)
print('要插入的数据数量为 %d' % len(contentList))
insert2DB = []
titleList.append("店铺的名称")
for r in contentList:
tempList = []
for i, value in enumerate(r):
# print(type(value))
tempList.append(value)
tempList.append(name[0].encode('utf-8'))
insert2DB.append(tempList)
if len(insert2DB) > 0:
sql.insertData(tt, titleList, insert2DB)
def callback_result(self, scan_result):
tmp = scan_result
for i in tmp['scan'].keys():
host = i
result = ''
try:
# result = u"ip地址:%s 主机名:%s ...... %s\n" %(host,tmp['scan'][host].get('hostnames','null'),tmp['scan'][host]['status'].get('state','null'))
# self.sqlTool.connectdb()
# print tmp['scan'][host].get('hostname','null')
# if 'osclass' in tmp['scan'][host].keys():
# result +=u"系统信息 ： %s %s %s 准确度:%s \n" % (str(tmp['scan'][host]['osclass'].get('vendor','null')),str(tmp['scan'][host]['osclass'].get('osfamily','null')),str(tmp['scan'][host]['osclass'].get('osgen','null')),str(tmp['scan'][host]['osclass'].get('accuracy','null')))
# print result
temphosts = str(host)
localtime = str(time.strftime("%Y-%m-%d %X", time.localtime()))
self.getlocationtool.add_work([temphosts])
try:
tempvendor = 'null'
temposfamily = 'null'
temposgen = 'null'
tempaccuracy = 'null'
if len(tmp['scan'][host]['osmatch']) > 0 and len(
tmp['scan'][host]['osmatch'][0]['osclass']) > 0:
tempvendor = str(
tmp['scan'][host]['osmatch'][0]['osclass'][0].get(
'vendor', 'null'))
temposfamily = str(
tmp['scan'][host]['osmatch'][0]['osclass'][0].get(
'osfamily', 'null'))
temposgen = str(
tmp['scan'][host]['osmatch'][0]['osclass'][0].get(
'osgen', 'null'))
tempaccuracy = str(
tmp['scan'][host]['osmatch'][0]['osclass'][0].get(
'accuracy', 'null'))
temphostname = ''
tempdecide = tmp['scan'][host].get('hostnames', [])
if len(tempdecide) > 0:
for y in tmp['scan'][host]['hostnames']:
temphostname += str(y.get('name', 'unknow')) + ' '
tempstate = str(tmp['scan'][host]['status'].get(
'state', 'null'))
# print temphosts,tempvendor,temposfamily,temposgen,tempaccuracy,localtime
# self.sqlTool.replaceinserttableinfo_byparams(table=self.config.iptable,select_params= ['ip','vendor','osfamily','osgen','accurate','updatetime','hostname','state'],insert_values= [(temphosts,tempvendor,temposfamily,temposgen,tempaccuracy,localtime,temphostname,tempstate)])
sqldatawprk = []
dic = {
"table":
self.config.iptable,
"select_params": [
'ip', 'vendor', 'osfamily', 'osgen', 'accurate',
'updatetime', 'hostname', 'state'
],
"insert_values":
[(temphosts, tempvendor, temposfamily, temposgen,
tempaccuracy, localtime, temphostname, tempstate)]
}
tempwprk = Sqldata.SqlData(
'replaceinserttableinfo_byparams', dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
except Exception, e:
print 'nmap system error d ' + str(e)
if 'tcp' in tmp['scan'][host].keys():
ports = tmp['scan'][host]['tcp'].keys()
for port in ports:
# portinfo = " port : %s name:%s state : %s product : %s version :%s script:%s \n" %(port,tmp['scan'][host]['tcp'][port].get('name',''),tmp['scan'][host]['tcp'][port].get('state',''), tmp['scan'][host]['tcp'][port].get('product',''),tmp['scan'][host]['tcp'][port].get('version',''),tmp['scan'][host]['tcp'][port].get('script',''))
tempport = str(port)
tempportname = str(tmp['scan'][host]['tcp'][port].get(
'name', ''))
tempportstate = str(tmp['scan'][host]['tcp'][port].get(
'state', ''))
tempproduct = str(tmp['scan'][host]['tcp'][port].get(
'product', ''))
tempportversion = str(
tmp['scan'][host]['tcp'][port].get('version', ''))
tempscript = SQLTool.decodestr(
str(tmp['scan'][host]['tcp'][port].get(
'script', {})))
# self.sqlTool.replaceinserttableinfo_byparams(table=self.config.porttable,select_params= ['ip','port','timesearch','state','name','product','version','script'],insert_values= [(temphosts,tempport,localtime,tempportstate,tempportname,tempproduct,tempportversion,tempscript)])
sqldatawprk = []
dic = {
"table":
self.config.porttable,
"select_params": [
'ip', 'port', 'timesearch', 'state', 'name',
'product', 'version', 'script', 'portnumber'
],
"insert_values":
[(temphosts, tempport, localtime, tempportstate,
tempportname, tempproduct, tempportversion,
tempscript, str(tempport))]
}
tempwprk = Sqldata.SqlData(
'replaceinserttableinfo_byparams', dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
self.portscan.add_work([
(tempportname, temphosts, tempport, tempportstate,
tempproduct, tempscript)
])
elif 'udp' in tmp['scan'][host].keys():
ports = tmp['scan'][host]['udp'].keys()
for port in ports:
# portinfo = " port : %s name:%s state : %s product : %s version :%s script:%s \n" %(port,tmp['scan'][host]['udp'][port].get('name',''),tmp['scan'][host]['udp'][port].get('state',''), tmp['scan'][host]['udp'][port].get('product',''),tmp['scan'][host]['udp'][port].get('version',''),tmp['scan'][host]['udp'][port].get('script',''))
# result = result + portinfo
tempport = str(port)
tempportname = str(tmp['scan'][host]['udp'][port].get(
'name', ''))
tempportstate = str(tmp['scan'][host]['udp'][port].get(
'state', ''))
tempproduct = str(tmp['scan'][host]['udp'][port].get(
'product', ''))
tempportversion = str(
tmp['scan'][host]['udp'][port].get('version', ''))
tempscript = str(tmp['scan'][host]['udp'][port].get(
'script', ''))
# self.sqlTool.replaceinserttableinfo_byparams(table=self.config.porttable,select_params= ['ip','port','timesearch','state','name','product','version','script'],insert_values= [(temphosts,tempport,localtime,tempportstate,tempportname,tempproduct,tempportversion,tempscript)])
sqldatawprk = []
dic = {
"table":
self.config.porttable,
"select_params": [
'ip', 'port', 'timesearch', 'state', 'name',
'product', 'version', 'script', 'portnumber'
],
"insert_values":
[(temphosts, tempport, localtime, tempportstate,
tempportname, tempproduct, tempportversion,
tempscript, str(tempport))]
}
tempwprk = Sqldata.SqlData(
'replaceinserttableinfo_byparams', dic)
sqldatawprk.append(tempwprk)
self.sqlTool.add_work(sqldatawprk)
except Exception, e:
print 'nmap error' + str(e)
# *-* coding:utf-8 *-*
import requests
import re
from bs4 import BeautifulSoup
from lxml import etree
from multiprocessing import Process, Queue
import SQLTool
MySql = SQLTool.MySQL()
class Proxies(object):
"""docstring for Proxies"""
def __init__(self, page=1):
self.proxies = []
self.verify_pro = []
self.page = page
self.headers = {
'Accept':
'*/*',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/45.0.2454.101 Safari/537.36',
'Accept-Encoding':
'gzip, deflate, sdch',
'Accept-Language':
'zh-CN,zh;q=0.8'
}
self.get_proxies()
self.get_proxies_nn()
# self.get_proxies_zdy()
