def setHeaders(self, headers):
if not isinstance(headers, dict):
Logger.error(
'PhantomJsResponse setHeaders() error: headers is not a dict...'
)
return
self.headers = headers
def saveData(self):
if configs.parse.file:
try:
dataList = []
try:
if os.path.exists(configs.parse.contentpath + configs.parse.contentfile):
dataList = readLinesFile(configs.parse.contentpath + configs.parse.contentfile)
for i in range(len(dataList)):
try:
self.addDataItem(json.loads(dataList[i]), self.primaryKey)
except Exception:
pass
except Exception:
pass
dataList = []
for item in self.dataList:
dataList.append(json.dumps(item, ensure_ascii = False))
writeLinesFile(configs.parse.contentpath + configs.parse.contentfile, dataList, method=configs.parse.savemethod)
except Exception as e:
Logger.error('an error occured while saving data to file...', e)
if configs.parse.mysql:
from espider.mysql import Mysql
keyList = []
for k in self.dataList[0]:
keyList.append(k)
mySql = Mysql(configs.mysql.table, keyList, primaryKey=self.primaryKey)
mySql.insertWithUpdate(self.dataList)
def setParser(self, parser):
if configs.spider.mode == 'override':
Logger.warning('Spider mode is override in configs. setParse() will be ignored. If you want to use update mode, change it in config_override')
return
if not isinstance(parser, BaseParser):
Logger.error('setParser() should have a BaseParser-like object input. Spider will scribe in override instead of update mode')
return
self.parser = parser
def getReqWithSel(self, request):
if not isinstance(request, urllib.request.Request):
Logger.error('SelePhan request error: please make sure request is a urllib.request.Request object...')
return None
url = request.full_url
self.driver.get(url)
response = PhantomJsResponse(self.driver.page_source, {'Content-Type':'text/html'})
return response
def loadProxy(self):
data = readLinesFile(configs.proxy.srcname)
if data == None:
Logger.critical('cannot load proxy list, espider is shuting down...')
exit(1)
proxyList = []
for i in range(len(data)):
proxyList.append(dict(zip(('type', 'ip', 'port', 'available', 'ping'), data[i].split('\t'))))
return proxyList
def catalogueUrlRecursion(self, url):
if configs.spider.catalogueLimit != 'inf':
if self.catalogueCount >= configs.spider.catalogueLimit:
return
url = urljoin(self.host, url)
urllistContent = []
urllistCatalogue = []
for i in range(configs.spider.retry):
response = self.httpHandler.getResponseByUrl(url)
if response == None:
Logger.warning(
'cannot get url %s. please check httphandler...' % url)
return
response = EsResponse(response)
try:
urllistCatalogue, urllistContent = self.getUrlList(response)
break
except ValueError:
Logger.critical(
'please verify your getUrlList() return 2 lists. espider is shutting down...'
)
exit(1)
except Exception as e:
Logger.error(
'an error occured in getUrlList(). if this take place often, please check your code'
)
self.httpHandler.nextHandler()
if i == configs.spider.retry - 1:
self.uncatchableUrlList.append(url)
self.saveUncatchableUrl()
if (len(urllistContent) != 0):
for item in urllistContent:
self.contentCount = self.contentCount + 1
if configs.spider.contentLimit != 'inf':
if self.contentCount > configs.spider.contentLimit:
break
if not keyValueInDictList('contentUrl', item,
self.contentDictList):
Logger.debug('discover content url %s' % item)
dictTemp = {}
dictTemp['contentUrl'] = item
self.contentDictList.append(dictTemp)
if len(urllistCatalogue) == 0:
return
else:
for item in urllistCatalogue:
if not item in self.catalogueUrl:
if configs.spider.catalogueLimit != 'inf':
if self.catalogueCount >= configs.spider.catalogueLimit:
return
Logger.info('get catalogue url %s' % item)
self.catalogueUrl.add(item)
self.catalogueCount = self.catalogueCount + 1
time.sleep(random.random() * configs.http.sleeptime)
self.catalogueUrlRecursion(item)
return
def backupUpdate(self):
if not os.path.exists(configs.spider.contentfilename):
return
if not os.path.exists(configs.spider.contentbackuppath):
os.makedirs(configs.spider.contentbackuppath)
now = datetime.now().strftime('%Y_%m_%d_%H_%M_%S_')
dstfilename = os.path.join(configs.spider.contentbackuppath, now + os.path.split(configs.spider.contentupdatefilename)[1])
try:
shutil.copy2(configs.spider.contentupdatefilename, dstfilename)
except IOError:
Logger.error('Cannot copy file to update path...')
def checkKeyList(self, keyList):
flag = True
if len(self.keyList) != len(keyList):
Logger.error('keyList length do not match...')
return False
for item in keyList:
if item not in self.keyList:
Logger.error('keyList element do not match')
flag = False
break
return flag
def getReqWithSel(self, request):
if not isinstance(request, urllib.request.Request):
Logger.error(
'SelePhan request error: please make sure request is a urllib.request.Request object...'
)
return None
url = request.full_url
self.driver.get(url)
response = PhantomJsResponse(self.driver.page_source,
{'Content-Type': 'text/html'})
return response
def checkKeyList(self, keyList):
flag = True
if len(self.keyList) != len(keyList):
Logger.error('keyList length do not match...')
return False
for item in keyList:
if item not in self.keyList:
Logger.error('keyList element do not match')
flag = False
break
return flag
def setParser(self, parser):
if configs.spider.mode == 'override':
Logger.warning(
'Spider mode is override in configs. setParse() will be ignored. If you want to use update mode, change it in config_override'
)
return
if not isinstance(parser, BaseParser):
Logger.error(
'setParser() should have a BaseParser-like object input. Spider will scribe in override instead of update mode'
)
return
self.parser = parser
def __init__(self, contentType, primaryKey = None, contentPath = configs.spider.contentdatapath, openMethod = 'rb', openEncoding = None):
if self.parserName == '':
Logger.warning('You should define parserName for your parser! Espider is shutting down...')
exit(1)
self.contentType = contentType
self.contentPath = contentPath
self.openMethod = openMethod
self.openEncoding = openEncoding
self.dataList = []
self.primaryValue = []
self.primaryKey = primaryKey
self.contentPath = contentPath
def __init__(self, response):
self.data = b''
self.headers = []
self.code = ''
if response == None:
return
if not isinstance(response, HTTPResponse):
Logger.error('EsRequest error: wrong type of response')
return
self.data = response.read()
self.headers = response.getheaders()
self.code = response.getcode()
self.url = response.geturl()
def loadProxy(self):
data = readLinesFile(configs.proxy.srcname)
if data == None:
Logger.critical(
'cannot load proxy list, espider is shuting down...')
exit(1)
proxyList = []
for i in range(len(data)):
proxyList.append(
dict(
zip(('type', 'ip', 'port', 'available', 'ping'),
data[i].split('\t'))))
return proxyList
def backupUpdate(self):
if not os.path.exists(configs.spider.contentfilename):
return
if not os.path.exists(configs.spider.contentbackuppath):
os.makedirs(configs.spider.contentbackuppath)
now = datetime.now().strftime('%Y_%m_%d_%H_%M_%S_')
dstfilename = os.path.join(
configs.spider.contentbackuppath,
now + os.path.split(configs.spider.contentupdatefilename)[1])
try:
shutil.copy2(configs.spider.contentupdatefilename, dstfilename)
except IOError:
Logger.error('Cannot copy file to update path...')
def catalogueUrlRecursion(self, url):
if configs.spider.catalogueLimit != 'inf':
if self.catalogueCount >= configs.spider.catalogueLimit:
return
url = urljoin(self.host, url)
urllistContent = []
urllistCatalogue = []
for i in range(configs.spider.retry):
response = self.httpHandler.getResponseByUrl(url)
if response == None:
Logger.warning('cannot get url %s. please check httphandler...' % url)
return
response = EsResponse(response)
try:
urllistCatalogue, urllistContent = self.getUrlList(response)
break
except ValueError:
Logger.critical('please verify your getUrlList() return 2 lists. espider is shutting down...')
exit(1)
except Exception as e:
Logger.error('an error occured in getUrlList(). if this take place often, please check your code')
self.httpHandler.nextHandler()
if i == configs.spider.retry - 1:
self.uncatchableUrlList.append(url)
self.saveUncatchableUrl()
if(len(urllistContent) != 0):
for item in urllistContent:
self.contentCount = self.contentCount + 1
if configs.spider.contentLimit != 'inf':
if self.contentCount > configs.spider.contentLimit:
break
if not keyValueInDictList('contentUrl', item, self.contentDictList):
Logger.debug('discover content url %s' % item)
dictTemp = {}
dictTemp['contentUrl'] = item
self.contentDictList.append(dictTemp)
if len(urllistCatalogue) == 0:
return
else:
for item in urllistCatalogue:
if not item in self.catalogueUrl:
if configs.spider.catalogueLimit != 'inf':
if self.catalogueCount >= configs.spider.catalogueLimit:
return
Logger.info('get catalogue url %s' % item)
self.catalogueUrl.add(item)
self.catalogueCount = self.catalogueCount + 1
time.sleep(random.random() * configs.http.sleeptime)
self.catalogueUrlRecursion(item)
return
def loadContentUpdateFileList(self):
if not os.path.exists(configs.spider.contentupdatefilename):
return []
dataList = readLinesFile(configs.spider.contentupdatefilename)
fileList = []
try:
for item in dataList:
if item.startswith('#'):
continue
data = item.split('\t')
fileList.append(data[3])
except IndexError:
Logger.error('Loading contentupdatefile error!')
return fileList
def startEspider(self):
if configs.spider.mode != 'override' and configs.spider.mode != 'update':
Logger.error(
'Please verify spider.mode is override or update in configs. Spider will run in default mode(override)'
)
if configs.spider.mode == 'update' and self.parser == None:
Logger.error(
'Spider cannot run in update mode without a correct function setParser() defined. '
)
Logger.info('Espider running in %s mode' %
('override' if self.parser == None else 'update'))
if self.parser != None:
# update mode
self.backupUpdate()
self.contentDictList = self.loadContentDictList()
Logger.info('start to get catalogue urls...')
if configs.spider.loadurllistfromfile:
self.loadCatalogueList()
self.contentDictList = self.loadContentDictList()
else:
self.catalogueUrlRecursion(self.startUrl)
writeLinesFile(configs.spider.cataloguefilename,
self.catalogueUrl,
method='w+')
count = 0
for item in self.contentDictList:
count = count + 1
MD5, filepath = self.contentHandler(item['contentUrl'], count)
item['filepath'] = filepath
if 'MD5' in item:
if self.parser == None:
item['update'] = 'disabled'
elif item['MD5'] == MD5:
item['update'] = 'false'
else:
item['update'] = 'true'
item['MD5'] = MD5
else:
if self.parser == None:
item['update'] = 'disabled'
else:
item['update'] = 'true'
item['MD5'] = MD5
self.saveContentUrlDictList()
self.saveContentUrlUpdate()
Logger.info('espider complete the task!')
def readLinesFile(filename, method = 'r'):
"""
Read from a file and extract each line to the element of a list.
"""
dataInLine = []
try:
with open(filename, method, encoding='utf8') as f:
data = f.readlines()
for i in range(len(data)):
data[i] = data[i].strip()
dataInLine = data
except OSError:
Logger.error('an errer occured when open %s' %filename)
return None
return dataInLine
def __sqlExecute(self,sql):
data = None
try:
connection = pymysql.connect(host = configs.mysql.host, port = configs.mysql.port, user = configs.mysql.user, password = configs.mysql.password, db = configs.mysql.db, charset = 'utf8')
try:
with connection.cursor() as cur:
cur.execute(sql)
data = cur.fetchall()
connection.commit()
except Exception:
Logger.error('sql statement execute error:%s' %sql)
finally:
connection.close()
except Exception:
Logger.error('mysql database open with error...')
return data
def writeLinesFile(filename, dataInLine, method = 'w'):
"""
Write a list to the file. One element to one line.
"""
if not isinstance(dataInLine, Iterable):
Logger.error('input illegal')
return
if not os.path.exists(os.path.split(filename)[0]):
os.makedirs(os.path.split(filename)[0])
dataInLine = [str(line) + '\n' for line in dataInLine]
try:
with open(filename, method, encoding = 'utf8') as f:
f.writelines(dataInLine)
except OSError:
Logger.error('an errer occured when open %s' %filename)
return
def createTable(self):
if not isinstance(self.keyList, list):
Logger.error('key list error when creating table %s' %self.table)
return
if len(self.keyList) == 0:
Logger.error('key list without an element in it, cannot create table %s' %self.table)
return
keyList = list()
for i in range(len(self.keyList)):
keyList.append(self.keyList[i] + ' VARCHAR(255)')
if self.primaryKey != None:
if self.primaryKey == self.keyList[0]:
keyList[0] = keyList[0] + ' PRIMARY KEY'
temp = ','.join(keyList)
sql = "CREATE TABLE IF NOT EXISTS %s(%s)" %(self.table, temp)
self.__sqlExecute(sql)
return
def createTable(self):
if not isinstance(self.keyList, list):
Logger.error('key list error when creating table %s' % self.table)
return
if len(self.keyList) == 0:
Logger.error(
'key list without an element in it, cannot create table %s' %
self.table)
return
keyList = list()
for i in range(len(self.keyList)):
keyList.append(self.keyList[i] + ' VARCHAR(255)')
if self.primaryKey != None:
if self.primaryKey == self.keyList[0]:
keyList[0] = keyList[0] + ' PRIMARY KEY'
temp = ','.join(keyList)
sql = "CREATE TABLE IF NOT EXISTS %s(%s)" % (self.table, temp)
self.__sqlExecute(sql)
return
def loadContentDictList(self):
if not os.path.exists(configs.spider.contentfilename):
return []
dataList = readLinesFile(configs.spider.contentfilename)
dataDictList = []
try:
for item in dataList:
if item.startswith('#'):
continue
t = {}
data = item.split('\t')
t['contentUrl'] = data[0]
t['MD5'] = data[1]
t['update'] = data[2]
t['filepath'] = data[3]
dataDictList.append(t)
except IndexError:
Logger.error('Loading contentfile error!')
return dataDictList
def loadContentDictList(self):
if not os.path.exists(configs.spider.contentfilename):
return []
dataList = readLinesFile(configs.spider.contentfilename)
dataDictList = []
try:
for item in dataList:
if item.startswith('#'):
continue
t = {}
data = item.split('\t')
t['contentUrl'] = data[0]
t['MD5'] = data[1]
t['update'] = data[2]
t['filepath'] = data[3]
dataDictList.append(t)
except IndexError:
Logger.error('Loading contentfile error!')
return dataDictList
def startEspider(self):
if configs.spider.mode != 'override' and configs.spider.mode != 'update':
Logger.error('Please verify spider.mode is override or update in configs. Spider will run in default mode(override)')
if configs.spider.mode == 'update' and self.parser == None:
Logger.error('Spider cannot run in update mode without a correct function setParser() defined. ')
Logger.info('Espider running in %s mode' %('override' if self.parser == None else 'update'))
if self.parser != None:
# update mode
self.backupUpdate()
self.contentDictList = self.loadContentDictList()
Logger.info('start to get catalogue urls...')
if configs.spider.loadurllistfromfile:
self.loadCatalogueList()
self.contentDictList = self.loadContentDictList()
else:
self.catalogueUrlRecursion(self.startUrl)
writeLinesFile(configs.spider.cataloguefilename, self.catalogueUrl, method='w+')
count = 0
for item in self.contentDictList:
count = count + 1
MD5, filepath = self.contentHandler(item['contentUrl'], count)
item['filepath'] = filepath
if 'MD5' in item:
if self.parser == None:
item['update'] = 'disabled'
elif item['MD5'] == MD5:
item['update'] = 'false'
else:
item['update'] = 'true'
item['MD5'] = MD5
else:
if self.parser == None:
item['update'] = 'disabled'
else:
item['update'] = 'true'
item['MD5'] = MD5
self.saveContentUrlDictList()
self.saveContentUrlUpdate()
Logger.info('espider complete the task!')
def select(self,selectKeyList = None):
temp = []
if selectKeyList == None:
temp = '*'
else:
if not isinstance(selectKeyList, list):
Logger.error('selectKeyList error because it is not a list...')
return None
temp = ','.join(selectKeyList)
sql = 'SELECT %s FROM %s' %(temp, self.table)
result = list(self.__sqlExecute(sql))
if selectKeyList == None:
sql = "select COLUMN_NAME from information_schema.COLUMNS where table_name = '%s' and table_schema = '%s'" %(self.table, configs.mysql.db)
selectKeyList = list(self.__sqlExecute(sql))
for i in range(len(selectKeyList)):
selectKeyList[i] = selectKeyList[i][0]
ret = []
for item in result:
ret.append(dict(zip(selectKeyList, item)))
return ret
def formProxy(self, count):
if len(self.proxyList) == 0:
self.proxy = None
return
if count >= len(self.proxyList):
Logger.error('SelePhan proxy form error:out of range in proxyList...')
self.proxy = None
return
proxy = self.proxyList[count]
ipport = proxy['ip'] + ':' + proxy['port']
proxyDict = {'proxyType':ProxyType.MANUAL}
if proxy['type'] == 'http':
proxyDict['httpProxy'] = ipport
elif proxy['type'] == 'socks':
proxyDict['socksProxy'] = ipport
else:
self.proxy = None
return
self.proxy = seleProxy(proxyDict)
return
def select(self, selectKeyList=None):
temp = []
if selectKeyList == None:
temp = '*'
else:
if not isinstance(selectKeyList, list):
Logger.error('selectKeyList error because it is not a list...')
return None
temp = ','.join(selectKeyList)
sql = 'SELECT %s FROM %s' % (temp, self.table)
result = list(self.__sqlExecute(sql))
if selectKeyList == None:
sql = "select COLUMN_NAME from information_schema.COLUMNS where table_name = '%s' and table_schema = '%s'" % (
self.table, configs.mysql.db)
selectKeyList = list(self.__sqlExecute(sql))
for i in range(len(selectKeyList)):
selectKeyList[i] = selectKeyList[i][0]
ret = []
for item in result:
ret.append(dict(zip(selectKeyList, item)))
return ret
def __sqlExecute(self, sql):
data = None
try:
connection = pymysql.connect(host=configs.mysql.host,
port=configs.mysql.port,
user=configs.mysql.user,
password=configs.mysql.password,
db=configs.mysql.db,
charset='utf8')
try:
with connection.cursor() as cur:
cur.execute(sql)
data = cur.fetchall()
connection.commit()
except Exception:
Logger.error('sql statement execute error:%s' % sql)
finally:
connection.close()
except Exception:
Logger.error('mysql database open with error...')
return data
def addDataItem(self, item, primaryKey):
itemtemp = OrderedDict()
for k,v in item.items():
if isinstance(v, list):
if len(v) == 0:
itemtemp[k] = ''
else:
itemtemp[k] = v[0]
else:
itemtemp[k] = v
if primaryKey != None and primaryKey in itemtemp:
if itemtemp[primaryKey] not in self.primaryValue:
if self.primaryKey == None:
self.primaryKey = primaryKey
elif self.primaryKey != primaryKey:
Logger.critical('different primary key found in returned data. espider is shutting down...')
exit(1)
self.primaryValue.append(itemtemp[primaryKey])
self.dataList.append(itemtemp)
return
self.dataList.append(itemtemp)
def formProxy(self, count):
if len(self.proxyList) == 0:
self.proxy = None
return
if count >= len(self.proxyList):
Logger.error(
'SelePhan proxy form error:out of range in proxyList...')
self.proxy = None
return
proxy = self.proxyList[count]
ipport = proxy['ip'] + ':' + proxy['port']
proxyDict = {'proxyType': ProxyType.MANUAL}
if proxy['type'] == 'http':
proxyDict['httpProxy'] = ipport
elif proxy['type'] == 'socks':
proxyDict['socksProxy'] = ipport
else:
self.proxy = None
return
self.proxy = seleProxy(proxyDict)
return
def __init__(self, startUrl=None):
self.startUrl = startUrl
path, file = os.path.split(configs.proxy.srcname)
if not os.path.exists(path):
os.makedirs(path)
if startUrl == None:
self.startUrl = 'http://www.baidu.com'
if configs.proxy.rescrab:
Logger.info('rescrab proxylist...')
self.getFreeProxy()
if configs.proxy.retest:
Logger.info('retest proxy list...')
self.testProxy()
if not os.path.exists(configs.proxy.srcname):
self.loadDefaultProxy()
else:
self.proxyList = self.loadProxy()
self.proxyList = list(
filter(lambda x: abs(int(x['available'])) == 1, self.proxyList))
self.proxyList = list(
filter(lambda x: float(x['ping']) < 2, self.proxyList))
if len(self.proxyList) == 0:
Logger.critical(
'There is no available proxy! espider is shuting down...')
exit(1)
self.proxyList.sort(
key=lambda x: 1000 if float(x['ping']) == -1 else float(x['ping']))
self.proxyCount = 0
def testProxy(self):
"""
Test the proxy connection performance with self.startUrl.
"""
req = urllib.request.Request(self.startUrl)
for k,v in configs.urlrequest.items():
if isinstance(v, list):
l = len(v)
v = v[randint(0, len(v) - 1)]
req.add_header(k,v)
Logger.info('test proxy list in %s' % configs.proxy.srcname)
data = readLinesFile(configs.proxy.srcname)
time.clock()
proxyList = []
for i in range(len(data)):
proxyList.append(dict(zip(('type', 'ip', 'port'), data[i].split('\t'))))
openner = urllib.request.build_opener(urllib.request.ProxyHandler({proxyList[i]['type']:proxyList[i]['ip'] + ':' + proxyList[i]['port']}), urllib.request.ProxyBasicAuthHandler())
try:
begin = time.clock()
openner.open(req, timeout=configs.proxy.timeout)
ping = time.clock() - begin
available = 1
Logger.info('proxy %s is good...' %proxyList[i]['ip'])
except Exception as e:
Logger.info('proxy %s is not available...' %proxyList[i]['ip'])
ping = -1
available = 0
proxyList[i]['available'] = available
proxyList[i]['ping'] = ping
dataset = []
for i in range(len(proxyList)):
dataset.append('%s\t%s\t%s\t%s\t%s' %(proxyList[i]['type'], proxyList[i]['ip'], proxyList[i]['port'], proxyList[i]['available'], proxyList[i]['ping']))
writeLinesFile(configs.proxy.srcname, dataset)
return
def getFreeProxy(self):
"""
Two different ways getting free proxy which can be configured in configs.
You can also define your own way of getting.
"""
Logger.info('get free proxy from the Internet...')
proxyList = []
if configs.proxy.proxysrc == 1:
for i in range(configs.proxy.srcpage):
Logger.info('get page %s...' %(i + 1))
req = urllib.request.Request('http://www.kuaidaili.com/free/inha/%s/' %(i + 1))
req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0')
data = urllib.request.urlopen(req).read().decode('utf-8')
proxy = re.findall('<tr>\s*?<td data-title="IP">(.*?)</td>\s*?<td data-title="PORT">(.*?)</td>\s*?<td data-title="匿名度">.*?</td>\s*?<td data-title="类型">(.*?)</td>', data)
for item in proxy:
if not [item[2].lower(), item[0], item[1]] in proxyList:
proxyList.append([item[2].lower(), item[0], item[1]])
if configs.proxy.proxysrc == 2:
for i in range(configs.proxy.srcpage):
Logger.info('get page %s...' %(i + 1))
req = urllib.request.Request('http://www.xicidaili.com/nn/%s' %(i + 1))
req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0')
data = urllib.request.urlopen(req).read().decode('utf-8')
proxy = re.findall('<td class="country"><img [\s\S]*?<td>(.*?)</td>\s*?<td>(.*?)</td>\s*?<td>[\s\S]*?</td>\s*?<td class="country">.*?</td>\s*?<td>(.*)</td>', data)
for item in proxy:
if not [item[2].lower(), item[0], item[1]] in proxyList:
proxyList.append([item[2].lower(), item[0], item[1]])
dataset = []
for item in proxyList:
dataset.append('%s\t%s\t%s\t-1\t-1' %(item[0], item[1], item[2]))
writeLinesFile(configs.proxy.srcname, dataset)
def checkUrlQuery(self):
if not isinstance(self.queryList, list) or len(self.queryList) == 0:
Logger.critical('Please define queryList as a non-empty list! Espider is shutting down...')
exit(1)
if not isinstance(self.parameterList, list) or len(self.parameterList) == 0:
Logger.critical('Please define parameterList as a non-empth list! Espider is shutting down...')
exit(1)
if not isinstance(self.extraParameter, OrderedDict):
Logger.critical('extraParameter should be OrderedDict! Espider is shutting down')
exit(1)
if len(self.queryList) != len(self.parameterList):
Logger.critical('Different length of queryList and parameterList, please make sure they match each other. Espider is shutting down...')
exit(1)
self.level = len(self.queryList)
def catalogueUrlRecursion(self, param, path, level):
if not os.path.exists(path):
os.makedirs(path)
Logger.info('(level %s)start to scrab param:%s' % (level, param))
if not isinstance(self.queryList[level - 1], list):
self.queryList[level - 1] = [self.queryList[level - 1]]
for query in self.queryList[level - 1]:
url = self.buildUrl(query, param)
url, headers = self.buildExtraHeaders(url)
response = self.httpHandler.getResponseByUrl(url, headers=headers)
data, type = self.contentResponseHandle(response)
with open(path + 'data_query=' + query + '.' + type,
'w+',
encoding='utf8') as f:
f.write(data)
if level == self.level:
return
try:
nextParamList = self.contentHandler(data)
except Exception:
Logger.error(
'an error occured in contentHandler(). If this take place often, please shut espider down...'
)
nextParamList = None
if nextParamList == None or nextParamList == []:
return
if not isinstance(nextParamList, list):
Logger.critical(
'contentHandler() should return a list. Espider is shutting down...'
)
exit(1)
if not isinstance(nextParamList[0], dict):
Logger.critical(
'contentHandler() should return list made by dict of each element. Espider is shutting down...'
)
exit(1)
writeLinesFile(path + 'param_query=' + query + '.txt',
nextParamList)
for nextParam in nextParamList:
for k, v in nextParam.items():
if k in self.parameterList[level]:
nextParamDict = dict(param)
nextParamDict[k] = v
nextPath = path + k + '=' + v + '/'
time.sleep(random.random() * configs.http.sleeptime)
self.catalogueUrlRecursion(nextParamDict, nextPath,
level + 1)
else:
pass
def insertWithUpdate(self, insertList):
if not isinstance(insertList, list):
Logger.error('insert list error because it is not a list...')
if len(insertList) == 0:
return
keyList = []
for k in insertList[0]:
keyList.append(k)
if not self.checkKeyList(keyList):
return
temp1 = ','.join(self.keyList)
for item in insertList:
valueList = []
for k in self.keyList:
valueList.append("'" + item[k] + "'")
temp2 = ','.join(valueList)
keyValueList = []
for i in range(1, len(self.keyList)):
keyValueList.append(self.keyList[i] + '=' + "VALUES(" + self.keyList[i] + ")")
temp3 = ','.join(keyValueList)
sql = "INSERT INTO %s(%s)VALUES(%s)ON DUPLICATE KEY UPDATE %s" %(self.table, temp1, temp2, temp3)
self.__sqlExecute(sql)
def insertWithUpdate(self, insertList):
if not isinstance(insertList, list):
Logger.error('insert list error because it is not a list...')
if len(insertList) == 0:
return
keyList = []
for k in insertList[0]:
keyList.append(k)
if not self.checkKeyList(keyList):
return
temp1 = ','.join(self.keyList)
for item in insertList:
valueList = []
for k in self.keyList:
valueList.append("'" + item[k] + "'")
temp2 = ','.join(valueList)
keyValueList = []
for i in range(1, len(self.keyList)):
keyValueList.append(self.keyList[i] + '=' + "VALUES(" +
self.keyList[i] + ")")
temp3 = ','.join(keyValueList)
sql = "INSERT INTO %s(%s)VALUES(%s)ON DUPLICATE KEY UPDATE %s" % (
self.table, temp1, temp2, temp3)
self.__sqlExecute(sql)
def __init__(self):
Logger.info('Espider %s initiating...' % self.espiderName)
if self.startUrl == '':
Logger.critical('Your espider should have a startUrl! Espider is shutting down...')
exit(1)
self.startUrl = urlunparse(urlparse(self.startUrl, 'http'))
if urlparse(self.startUrl).hostname == None:
Logger.critical('Illegal url! Please make sure url like "http://www.baidu.com". Espider will be closed...')
exit(1)
self.host = urlparse(self.startUrl).scheme + '://' + urlparse(self.startUrl).hostname
self.checkUrlQuery()
self.httpHandler = HttpHandler(self.host)
def catalogueUrlRecursion(self, param, path, level):
if not os.path.exists(path):
os.makedirs(path)
Logger.info('(level %s)start to scrab param:%s' % (level, param))
if not isinstance(self.queryList[level - 1], list):
self.queryList[level - 1] = [self.queryList[level - 1]]
for query in self.queryList[level - 1]:
url = self.buildUrl(query, param)
url, headers = self.buildExtraHeaders(url)
response = self.httpHandler.getResponseByUrl(url, headers=headers)
data, type = self.contentResponseHandle(response)
with open(path + 'data_query=' + query + '.' + type, 'w+', encoding='utf8') as f:
f.write(data)
if level == self.level:
return
try:
nextParamList = self.contentHandler(data)
except Exception:
Logger.error('an error occured in contentHandler(). If this take place often, please shut espider down...')
nextParamList = None
if nextParamList == None or nextParamList == []:
return
if not isinstance(nextParamList, list):
Logger.critical('contentHandler() should return a list. Espider is shutting down...')
exit(1)
if not isinstance(nextParamList[0], dict):
Logger.critical('contentHandler() should return list made by dict of each element. Espider is shutting down...')
exit(1)
writeLinesFile(path + 'param_query=' + query + '.txt', nextParamList)
for nextParam in nextParamList:
for k,v in nextParam.items():
if k in self.parameterList[level]:
nextParamDict = dict(param)
nextParamDict[k] = v
nextPath = path + k + '=' + v + '/'
time.sleep(random.random() * configs.http.sleeptime)
self.catalogueUrlRecursion(nextParamDict, nextPath, level + 1)
else:
pass
def getResponseByUrl(self, url, headers={}):
"""
url is the website you want.
headers is the dict you add dynamicly apart from that in configs
"""
begin = time.clock()
if urlparse(url).hostname == None:
Logger.error('url of request illegal! which is %s' % url)
return None
req = urllib.request.Request(url)
for k, v in configs.urlrequest.items():
if isinstance(v, list):
l = len(v)
v = v[randint(0, len(v) - 1)]
req.add_header(k, v)
for k, v in headers.items():
req.add_header(k, v)
flag = False
for i in range(configs.http.retry):
Logger.debug('%s attempt' % (i + 1))
try:
if self.selephan != None:
response = self.selephan.getReqWithSel(req)
if response == None:
continue
else:
flag = True
break
if self.proxy != None:
response = self.proxy.getReqWithProxy(
req, timeout=configs.proxy.timeout)
if response == None:
continue
else:
flag = True
break
response = urllib.request.urlopen(req,
timeout=configs.http.timeout)
flag = True
break
except Exception as e:
continue
end = time.clock()
Logger.debug('HTTP request time: %ss' % (end - begin))
if flag:
return response
else:
return None
def startEspider(self):
Logger.info('starting espider...')
paramList = readLinesFile(configs.spider.contentdatapath + 'param.txt')
if paramList == None:
Logger.critical('You should create starting parameters in %s' % (configs.spider.contentdatapath + 'param.txt'))
exit(1)
for i in range(len(paramList)):
paramList[i] = json.loads(paramList[i])
for k,v in paramList[i].items():
if k in self.parameterList[0]:
param = {}
param[k] = v
path = configs.spider.contentdatapath + k + '=' + v + '/'
self.catalogueUrlRecursion(param, path, 1)
else:
Logger.error('param.txt gives an incorrect key compared to self.paramterList...')
def checkUrlQuery(self):
if not isinstance(self.queryList, list) or len(self.queryList) == 0:
Logger.critical(
'Please define queryList as a non-empty list! Espider is shutting down...'
)
exit(1)
if not isinstance(self.parameterList, list) or len(
self.parameterList) == 0:
Logger.critical(
'Please define parameterList as a non-empth list! Espider is shutting down...'
)
exit(1)
if not isinstance(self.extraParameter, OrderedDict):
Logger.critical(
'extraParameter should be OrderedDict! Espider is shutting down'
)
exit(1)
if len(self.queryList) != len(self.parameterList):
Logger.critical(
'Different length of queryList and parameterList, please make sure they match each other. Espider is shutting down...'
)
exit(1)
self.level = len(self.queryList)
def __init__(self):
Logger.info('Espider %s initiating...' % self.espiderName)
if self.startUrl == '':
Logger.critical(
'Your espider should have a startUrl! Espider is shutting down...'
)
exit(1)
self.startUrl = urlunparse(urlparse(self.startUrl, 'http'))
if urlparse(self.startUrl).hostname == None:
Logger.critical(
'Illegal url! Please make sure url like "http://www.baidu.com". Espider will be closed...'
)
exit(1)
self.host = urlparse(self.startUrl).scheme + '://' + urlparse(
self.startUrl).hostname
self.checkUrlQuery()
self.httpHandler = HttpHandler(self.host)
def getResponseByUrl(self, url, headers={}):
"""
url is the website you want.
headers is the dict you add dynamicly apart from that in configs
"""
begin = time.clock()
if urlparse(url).hostname == None:
Logger.error('url of request illegal! which is %s' %url)
return None
req = urllib.request.Request(url)
for k,v in configs.urlrequest.items():
if isinstance(v, list):
l = len(v)
v = v[randint(0, len(v) - 1)]
req.add_header(k,v)
for k,v in headers.items():
req.add_header(k,v)
flag = False
for i in range(configs.http.retry):
Logger.debug('%s attempt' %(i+1))
try:
if self.selephan != None:
response = self.selephan.getReqWithSel(req)
if response == None:
continue
else:
flag = True
break
if self.proxy != None:
response= self.proxy.getReqWithProxy(req, timeout=configs.proxy.timeout)
if response == None:
continue
else:
flag = True
break
response = urllib.request.urlopen(req, timeout=configs.http.timeout)
flag = True
break
except Exception as e:
continue
end = time.clock()
Logger.debug('HTTP request time: %ss' %(end - begin))
if flag:
return response
else:
return None
def __init__(self):
Logger.info('espider %s initiating...' % self.espiderName)
if self.startUrl == '' or self.espiderName == '':
Logger.critical('Your espider should have an espiderName and a startUrl! Espider is shutting down...')
exit(1)
self.startUrl = urlunparse(urlparse(self.startUrl, 'http'))
if urlparse(self.startUrl).hostname == None:
Logger.critical('Illegal url! Please make sure url like "http://www.baidu.com". Espider will be closed...')
exit(1)
self.host = urlparse(self.startUrl).scheme + '://' + urlparse(self.startUrl).hostname
self.httpHandler = HttpHandler(self.host)
if not os.path.exists(configs.spider.pipelinepath):
os.makedirs(configs.spider.pipelinepath)
self.catalogueUrl = set()
self.catalogueCount = 0
self.contentCount = 0
self.contentDictList = []
self.uncatchableUrlList = []
def startEspider(self):
Logger.info('starting espider...')
paramList = readLinesFile(configs.spider.contentdatapath + 'param.txt')
if paramList == None:
Logger.critical('You should create starting parameters in %s' %
(configs.spider.contentdatapath + 'param.txt'))
exit(1)
for i in range(len(paramList)):
paramList[i] = json.loads(paramList[i])
for k, v in paramList[i].items():
if k in self.parameterList[0]:
param = {}
param[k] = v
path = configs.spider.contentdatapath + k + '=' + v + '/'
self.catalogueUrlRecursion(param, path, 1)
else:
Logger.error(
'param.txt gives an incorrect key compared to self.paramterList...'
)
def testProxy(self):
"""
Test the proxy connection performance with self.startUrl.
"""
req = urllib.request.Request(self.startUrl)
for k, v in configs.urlrequest.items():
if isinstance(v, list):
l = len(v)
v = v[randint(0, len(v) - 1)]
req.add_header(k, v)
Logger.info('test proxy list in %s' % configs.proxy.srcname)
data = readLinesFile(configs.proxy.srcname)
time.clock()
proxyList = []
for i in range(len(data)):
proxyList.append(
dict(zip(('type', 'ip', 'port'), data[i].split('\t'))))
openner = urllib.request.build_opener(
urllib.request.ProxyHandler({
proxyList[i]['type']:
proxyList[i]['ip'] + ':' + proxyList[i]['port']
}), urllib.request.ProxyBasicAuthHandler())
try:
begin = time.clock()
openner.open(req, timeout=configs.proxy.timeout)
ping = time.clock() - begin
available = 1
Logger.info('proxy %s is good...' % proxyList[i]['ip'])
except Exception as e:
Logger.info('proxy %s is not available...' %
proxyList[i]['ip'])
ping = -1
available = 0
proxyList[i]['available'] = available
proxyList[i]['ping'] = ping
dataset = []
for i in range(len(proxyList)):
dataset.append('%s\t%s\t%s\t%s\t%s' %
(proxyList[i]['type'], proxyList[i]['ip'],
proxyList[i]['port'], proxyList[i]['available'],
proxyList[i]['ping']))
writeLinesFile(configs.proxy.srcname, dataset)
return
def getFreeProxy(self):
"""
Two different ways getting free proxy which can be configured in configs.
You can also define your own way of getting.
"""
Logger.info('get free proxy from the Internet...')
proxyList = []
if configs.proxy.proxysrc == 1:
for i in range(configs.proxy.srcpage):
Logger.info('get page %s...' % (i + 1))
req = urllib.request.Request(
'http://www.kuaidaili.com/free/inha/%s/' % (i + 1))
req.add_header(
'User-Agent',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0'
)
data = urllib.request.urlopen(req).read().decode('utf-8')
proxy = re.findall(
'<tr>\s*?<td data-title="IP">(.*?)</td>\s*?<td data-title="PORT">(.*?)</td>\s*?<td data-title="匿名度">.*?</td>\s*?<td data-title="类型">(.*?)</td>',
data)
for item in proxy:
if not [item[2].lower(), item[0], item[1]] in proxyList:
proxyList.append([item[2].lower(), item[0], item[1]])
if configs.proxy.proxysrc == 2:
for i in range(configs.proxy.srcpage):
Logger.info('get page %s...' % (i + 1))
req = urllib.request.Request('http://www.xicidaili.com/nn/%s' %
(i + 1))
req.add_header(
'User-Agent',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0'
)
data = urllib.request.urlopen(req).read().decode('utf-8')
proxy = re.findall(
'<td class="country"><img [\s\S]*?<td>(.*?)</td>\s*?<td>(.*?)</td>\s*?<td>[\s\S]*?</td>\s*?<td class="country">.*?</td>\s*?<td>(.*)</td>',
data)
for item in proxy:
if not [item[2].lower(), item[0], item[1]] in proxyList:
proxyList.append([item[2].lower(), item[0], item[1]])
dataset = []
for item in proxyList:
dataset.append('%s\t%s\t%s\t-1\t-1' % (item[0], item[1], item[2]))
writeLinesFile(configs.proxy.srcname, dataset)
def __init__(self):
Logger.info('espider %s initiating...' % self.espiderName)
if self.startUrl == '' or self.espiderName == '':
Logger.critical(
'Your espider should have an espiderName and a startUrl! Espider is shutting down...'
)
exit(1)
self.startUrl = urlunparse(urlparse(self.startUrl, 'http'))
if urlparse(self.startUrl).hostname == None:
Logger.critical(
'Illegal url! Please make sure url like "http://www.baidu.com". Espider will be closed...'
)
exit(1)
self.host = urlparse(self.startUrl).scheme + '://' + urlparse(
self.startUrl).hostname
self.httpHandler = HttpHandler(self.host)
if not os.path.exists(configs.spider.pipelinepath):
os.makedirs(configs.spider.pipelinepath)
self.catalogueUrl = set()
self.catalogueCount = 0
self.contentCount = 0
self.contentDictList = []
self.uncatchableUrlList = []
def __init__(self, startUrl = None):
self.startUrl = startUrl
path, file = os.path.split(configs.proxy.srcname)
if not os.path.exists(path):
os.makedirs(path)
if startUrl == None:
self.startUrl = 'http://www.baidu.com'
if configs.proxy.rescrab:
Logger.info('rescrab proxylist...')
self.getFreeProxy()
if configs.proxy.retest:
Logger.info('retest proxy list...')
self.testProxy()
if not os.path.exists(configs.proxy.srcname):
self.loadDefaultProxy()
else:
self.proxyList = self.loadProxy()
self.proxyList = list(filter(lambda x:abs(int(x['available'])) == 1, self.proxyList))
self.proxyList = list(filter(lambda x:float(x['ping']) < 2, self.proxyList))
if len(self.proxyList) == 0:
Logger.critical('There is no available proxy! espider is shuting down...')
exit(1)
self.proxyList.sort(key = lambda x:1000 if float(x['ping']) == -1 else float(x['ping']))
self.proxyCount = 0
def getUrlList(self, response):
Logger.critical('getUrlList() without override! espider is shuting down...')
exit(1)
def setHeaders(self, headers):
if not isinstance(headers, dict):
Logger.error('PhantomJsResponse setHeaders() error: headers is not a dict...')
return
self.headers = headers
def setData(self, data):
if not isinstance(data, str):
Logger.error('PhantomJsResponse setData() error: data is not a str...')
return
self.data = bytes(data, encoding='utf8')
def contentHandler(self, url, count):
url = urljoin(self.host, url)
Logger.info('(%s%%)get content data from %s' % (round(100 * count / len(self.contentDictList), 2), url))
data = None
type = ''
name = None
for i in range(configs.spider.retry):
response = self.httpHandler.getResponseByUrl(url)
if response == None:
Logger.warning('cannot get url %s. please check httphandler...' % url)
return ('disabled', 'disabled')
response = EsResponse(response)
try:
data, type = self.contentResponseHandle(response)
if data == None:
Logger.debug('data == None')
raise Exception
name = self.contentFileName(response)
except Exception:
Logger.error('an error occured in getUrlList(). if this take place very often, please check your code')
self.httpHandler.nextHandler()
if i == configs.spider.retry - 1:
self.uncatchableUrlList.append(url)
self.saveUncatchableUrl()
continue
break
if data == None:
return ('disabled', 'disabled')
if name == None:
name = '%s.' % count + type
if not os.path.exists(configs.spider.contentdatapath):
os.makedirs(configs.spider.contentdatapath)
if self.parser == None:
MD5 = buildMD5String(data)
else:
try:
parsedData = '%s' %self.parser.parseContent(data)
MD5 = buildMD5String(parsedData)
except Exception:
Logger.error('An error occured in parseContent()! Please check your code. Espider will use the whole file as update md5')
MD5 = buildMD5String(data)
filepath = configs.spider.contentdatapath + name
try:
if type == 'html' or type == 'xml' or type == 'json' or type == 'js' or type == 'css':
with open(filepath, 'w+', encoding='utf8') as f:
f.write(data)
return (MD5, filepath)
if type == 'jpg' or type == 'tif' or type == 'ico' or type == 'png' or type == 'bmp' or type == 'mp3' or type == 'avi' or type == 'mp4':
with open(filepath, 'wb+') as f:
f.write(data)
return (MD5, filepath)
with open(filepath, 'wb+') as f:
f.write(data)
except OSError:
Logger.error('anerrer occured when open %s' % configs.spider.contentdatapath + name)
return (MD5, filepath)
def contentHandler(self, data):
Logger.critical('contentHandler() without override! espider is shuting down...')
exit(1)
