def __init__(self):
self.confile = r"..\conf\test.conf"
self.threadnum = 10
if os.path.exists(self.confile):
conf = confhelper.ConfHelper(self.confile)
thread = conf.GetConfig("others", "threadnum")
if thread <> None and thread.isdigit():
self.threadnum = int(thread)
def GetPorts(self):
conf = confhelper.ConfHelper(self.confile)
scanports = conf.GetSectionConfig("scanports")
if scanports.has_key("ips"):
ips = scanports.get("ips").split(',')
if scanports.has_key("ports"):
ports = scanports.get("ports").split(',')
ports = [int(item) for item in ports]
return ips, ports
def TestSqlitedb(secname):
conf = confhelper.ConfHelper()
dbinfo = conf.GetSectionConfig(secname)
mydb = SqlHelper(db=dbinfo.get("sqlitetest"), dbtype='sqlite')
sqlstr = "insert into company(id,name,age) values(%d,'%s',%d)" % (
2, "abddd", 8)
mydb.ExecNonQuery(sqlstr)
resList = mydb.ExecQuery("select * from company")
print resList
def TestMssqldb(secname):
conf = confhelper.ConfHelper()
dbinfo = conf.GetSectionConfig(secname)
mydb = SqlHelper(host=dbinfo.get("mssqlserver"),
user=dbinfo.get("mssqluser"),
pwd=dbinfo.get("mssqlpwd"),
db=dbinfo.get("testdb"),
dbtype='mssql')
resList = mydb.ExecQuery("SELECT * FROM test")
print resList
def SaveToDB(ymon,day,seq,name,title,url):
conf = confhelper.ConfHelper()
dbinfo = conf.GetSectionConfig("mysqldb")
mydb = sqlhelper.SqlHelper(host=dbinfo.get("mysqlhost"), user=dbinfo.get("mysqluser"), pwd=dbinfo.get("mysqlpwd"), db=dbinfo.get("mysqldb"),dbtype='mysql')
sqlstr = "insert into huodong(year,month,day,seq,name,title,url) select %d,%d,%d,%d,'%s','%s','%s' from DUAL where not exists (select 1 from huodong where year=%d and month=%d and day = %d and url='%s')" \
% (ymon.get("year"),ymon.get("month"),day,seq,name,title,url,ymon.get("year"),ymon.get("month"),day,url)
print sqlstr
mydb.ExecNonQuery(sqlstr)
sqlstr = "update huodong set name='%s',title='%s' where year=%d and month=%d and day = %d and url='%s'" \
% (name,title,ymon.get("year"),ymon.get("month"),day,url)
print sqlstr
mydb.ExecNonQuery(sqlstr)
def TestMysqldb(secname):
conf = confhelper.ConfHelper()
dbinfo = conf.GetSectionConfig(secname)
print dbinfo
mydb = SqlHelper(host=dbinfo.get("mysqlhost"),
user=dbinfo.get("mysqluser"),
pwd=dbinfo.get("mysqlpwd"),
db=dbinfo.get("mysqldb"),
dbtype='mysql')
sqlstr = "insert into urls(name,url) values('%s','%s')" % ("name", "url")
mydb.ExecNonQuery(sqlstr)
resList = mydb.ExecQuery("SELECT * FROM urls")
print resList
def __init__(self,
url=r'http://www.baidu.com',
headers={},
outfile=r'd:\temp\log.txt',
proxies={},
timeout=3):
self.confile = r"..\conf\test.conf"
if os.path.exists(self.confile):
self.conf = confhelper.ConfHelper(self.confile)
self.confs = self.conf.GetAllConfig()
if not (url.startswith('http://')) and not (
url.startswith('https://')):
url = 'http://' + url
#代理格式 http://user:password@host/
if len(proxies) == 0 and self.confs.has_key(
"username") and self.confs.has_key(
"password") and self.confs.has_key("httpserver"):
proxies['http'] = "http://%s:%s@%s" % (
self.confs.get("username"), self.confs.get("password"),
self.confs.get("httpserver"))
proxies['https'] = "http://%s:%s@%s" % (
self.confs.get("username"), self.confs.get("password"),
self.confs.get("httpsserver"))
self.proxies = proxies
#SOCKS 代理格式 socks5://user:pass@host:port
'''proxies = {
'http': 'socks5://user:pass@host:port',
'https': 'socks5://user:pass@host:port'
}'''
if len(headers) == 0:
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding': 'gzip,deflate,sdch',
'Accept-Language': 'zh-CN,zh;q=0.8',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded'
}
self.headers = headers
self.url = url
if outfile == "" and self.confs.has_key("logpath"):
outfile = self.confs.get("logpath")
self.outfile = outfile
self.session = requests.session()
self.session.proxies = self.proxies
if self.confs.has_key("theadnum") and self.confs.get(
"theadnum").isdigit():
self.theadnum = int(self.confs.get("theadnum"))
else:
self.theadnum = 10
def ZhuhaiHuWai(ymon):
huodong = confhelper.ConfHelper().GetConfig("zhhuwai","huodong")
t = timehelper.TimeHelper()
url = huodong % ("%d%02d" % (ymon.get("year"),ymon.get("month")))
print url
d=pq(url=url)
td = d("#z_tb_ecal tr td")
#div = d("#z_tb_ecal tr td div")
cnt = td.length
for i in range(0,cnt):
day = td.eq(i).find('div').text()
tda = td.eq(i).find('p a')
tdacnt = tda.length
for j in range(0,tdacnt):
name = tda.eq(j).text()
href = tda.eq(j).attr.href
title = tda.eq(j).attr.title
SaveToDB(ymon,int(day),j,name,title,href)