def __init__(self,host,port,username,password):
self.ssh=None
self.sftp=None
self.ignorefolder=[]
self.faillist=[]
self.localroot='./'
self.__sftpconn(host,username,password)
#初始化日志
self.log=kl_log.kl_log('kl_sftp')
'''
采集配置型号
'''
import kl_http,kl_db,os,json,kl_log
from postdata import postdata
addnum=0
http=kl_http.kl_http()
log=kl_log.kl_log('brand')
db=kl_db.mysql({
'host':'localhost',
'user':'******',
'passwd':'adminrootkl',
'db':'qiche',
'prefix':'kl_',
'charset':'utf8'
})
http.autoUserAgent=True
http.setheaders('''\
Host:www.epicc.com.cn
Origin:http://www.epicc.com.cn
Referer:http://www.epicc.com.cn/ecar/proposal/normalProposal
User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36
X-Requested-With:XMLHttpRequestContent-Type: application/x-www-form-urlencoded\
''')
try:
url='http://www.epicc.com.cn/ecar/car/carModel/getCarModelFromJYDB'
brandlist=db.table('4shoudong').where({'status':0}).order('id asc').getarr()
for i in brandlist:
tjdata=postdata['peizhixinghao'].replace('[GROUPID]',i['groupId'])
tjdata=tjdata.replace('[ENGINEDESC]',i['engineDesc'])
r=http.posturl(url,tjdata)
import sys,re,random,os,threading,time,_thread
sys.path.append('../../lib/')
import kl_http,kl_db,kl_reg,kl_progress,kl_log
from urllib.parse import urlparse
log=kl_log.kl_log('bokedaquan')
regex=kl_reg
http=kl_http.kl_http()
mylock = _thread.allocate_lock()#线程锁
#mylock.acquire() #Get the lock
#mylock.release() #Release the lock.
progress=kl_progress.kl_progress('正在采集中')
progress.start()
#最大线程
maxthread=10
threadnum=0
#是否用代理
isproxy=False
http.setproxy('','','127.0.0.1:8087')
db=kl_db.mysql({
'host':'localhost',
'user':'******',
'passwd':'adminrootkl',
'db':'bokedaquan',
'prefix':'kl_',
'charset':'utf8'
})
f=open('../proxy/proxy.txt','r')
s=f.read()
f.close()
import sys, re, random, os, threading, time, _thread
import kl_http, kl_db, kl_reg, kl_progressbar, kl_log
from kl_print import *
from urllib.parse import urlparse
log = kl_log.kl_log('spider')
regex = kl_reg
http = kl_http.kl_http()
#mylock = _thread.allocate_lock()#线程锁
#self.mylock.acquire() #Get the lock
#self.mylock.release() #Release the lock.
http.setproxy('', '', '127.0.0.1:8087')
db = kl_db.mysql({
'host': 'localhost',
'user': '******',
'passwd': 'adminrootkl',
'db': 'bokedaquan',
'prefix': 'kl_',
'charset': 'utf8'
})
proxypath = '../tool/proxy/proxy.txt'
proxylist = []
if os.path.exists(proxypath):
f = open(proxypath, 'r')
s = f.read()
f.close()
proxylist = s.splitlines()
class urlspider(object):
"""docstring for urlspider"""
def __init__(self, zip_dest_name, b='w'):
self.log = kl_log.kl_log('kl_zip')
zipfile.ZipFile.__init__(self, zip_dest_name, b)
self.ignore = []
import sys, re, random, os, threading, time, _thread
sys.path.append('../../lib/')
import kl_http, kl_db, kl_reg, kl_progress, kl_log
from urllib.parse import urlparse
log = kl_log.kl_log('bokedaquan')
regex = kl_reg
http = kl_http.kl_http()
mylock = _thread.allocate_lock() #线程锁
#mylock.acquire() #Get the lock
#mylock.release() #Release the lock.
progress = kl_progress.kl_progress('正在采集中')
progress.start()
#最大线程
maxthread = 10
threadnum = 0
#是否用代理
isproxy = False
http.setproxy('', '', '127.0.0.1:8087')
db = kl_db.mysql({
'host': 'localhost',
'user': '******',
'passwd': 'adminrootkl',
'db': 'bokedaquan',
'prefix': 'kl_',
'charset': 'utf8'
})
f = open('../proxy/proxy.txt', 'r')
s = f.read()
f.close()
import sys,re,random,os,threading,time,_thread
import kl_http,kl_db,kl_reg,kl_progressbar,kl_log
from kl_print import *
from urllib.parse import urlparse
log=kl_log.kl_log('spider')
regex=kl_reg
http=kl_http.kl_http()
#mylock = _thread.allocate_lock()#线程锁
#self.mylock.acquire() #Get the lock
#self.mylock.release() #Release the lock.
http.setproxy('','','127.0.0.1:8087')
db=kl_db.mysql({
'host':'localhost',
'user':'******',
'passwd':'adminrootkl',
'db':'bokedaquan',
'prefix':'kl_',
'charset':'utf8'
})
proxypath='../tool/proxy/proxy.txt'
proxylist=[]
if os.path.exists(proxypath):
f=open(proxypath,'r')
s=f.read()
f.close()
proxylist=s.splitlines()
class urlspider(object):
"""docstring for urlspider"""
def __init__(self, arg):
def __init__(self,zip_dest_name, b='w'):
self.log=kl_log.kl_log('kl_zip')
zipfile.ZipFile.__init__(self,zip_dest_name,b)
self.ignore=[]
"""
查询品牌
"""
import kl_http, kl_db, os, json, kl_log
from postdata import postdata
addnum = 0
http = kl_http.kl_http()
log = kl_log.kl_log("brand")
db = kl_db.mysql(
{"host": "localhost", "user": "******", "passwd": "adminrootkl", "db": "qiche", "prefix": "kl_", "charset": "utf8"}
)
# 取brand数据
url = "http://www.epicc.com.cn/ecar/car/carModel/getCarModelFromJYDB"
http.setheaders(
"""\
Host:www.epicc.com.cn
Origin:http://www.epicc.com.cn
Referer:http://www.epicc.com.cn/ecar/proposal/normalProposal
User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36
X-Requested-With:XMLHttpRequest\
"""
)
# 查询品牌#################################################
try:
for i in range(26):
code = chr(i + ord("A"))
tjdata = postdata["brand"].replace("[CODE]", code)
r = http.posturl(url, tjdata)
content = ""