def _generate_seed_url(self):
"""
generate all url to visit
"""
# from page 1 to anypage which < 200
# 从数据库添加
global Dao
Dao = Dao()
self._seed_url = Dao._get_url_by_id(self.source_id)
def _video_dao(self):
dao = Dao()
#表中是否已有记录
query_sql = "SELECT * FROM merchant WHERE url = '{}'".format(
self.__ne_detail["url"])
if dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(
self.__ne_detail["name"]))
return
#数据插入操作
dao.execute_dmls(self._insert_merchant())
def __init__(self,
LT1,
LG1,
LT2,
LG2,
cityname,
cityid,
cityenname,
name,
shopId=0,
categoryId=0):
threading.Thread.__init__(self, name=name)
self.cityid = cityid
self.shopId = shopId
self.categoryId = categoryId
self.Lat1 = LT1
self.Lat2 = LT2
self.Long1 = LG1
self.Long2 = LG2
self.city_name = cityname
self.values = {
'promoId': '0',
'shopType': '',
'categoryId': '',
'sortMode': '2',
'shopSortItem': '1',
'keyword': '',
'searchType': '1',
'branchGroupId': '0',
'shippingTypeFilterValue': '0',
'page': '1'
}
self.values["cityId"] = cityid
self.values["cityEnName"] = cityenname
self.url = "http://www.dianping.com/search/map/ajax/json"
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0',
'Referer':
'http://www.dianping.com/search/map/category/{}/0'.format(cityid)
}
self.dao = Dao()
self.query_sql = "SELECT shopType ,categoryId,NAME FROM category WHERE categoryId <> shopType AND categoryId <>'None' "
self.result = self.dao.execute_query(self.query_sql)
self.query_sql2 = "SELECT shopId FROM shop_bean where city_name ='{}'".format(
cityname)
self.result2 = self.dao.execute_query(self.query_sql2)
self.shopIds = []
if self.result2 is not None:
for shopid in self.result2:
self.shopIds.append(shopid[0])
def start(self):
if self._print_status:
print('start mining #{} ...'.format(self._hastag))
dao = Dao()
timeStamp_tweet_list = dao.select(
'*', "manager",
"hastag = '{}' ORDER BY 'timeStamp'".format(self._hastag))
lastTweetTimeStamp = int(timeStamp_tweet_list[-1]['timeStamp'])
if lastTweetTimeStamp > self.START_TIMESTAMP:
self.mining(lastTweetTimeStamp, self.FINISH_TIMESTAMP)
else:
self.mining(self.START_TIMESTAMP, self.FINISH_TIMESTAMP)
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
for single_url in seed_url:
# # 获取html源代码
# html = self.get_page_content_str(single_url)
#
# #使用哪个方法进行分析
# self._extract_data(html)
# dao=Dao()
# insert_sql =" INSERT INTO fetch_list (source_id, url,times,page,STATUS) VALUE(99,'{}',0,0,0)".format(single_url)
# dao.execute_dmls(insert_sql)
dao = Dao()
update_sql = " UPDATE fetch_list2 SET times = times+1 WHERE url = '{}'and source_id = 98 ".format(
single_url[0])
dao.execute_dmls(update_sql)
self._now_url = single_url[0]
html = self.get_page_content_str(single_url[0])
self._extract_data2(html)
from DAO import Dao
import sys
from extract_tweet_ids import HastagMining
import time
from tweet_from_id import TweetFromID
from models import Tweet,User
from sentiment_analyze import Linguakit
from utils import printError, limparTexto
s1 = sys.argv[1]
if s1 == 'newHashtag':
candidato, hastag = sys.argv[2], sys.argv[3]
dao = Dao()
r = dao.insert('manager', ['hastag', 'idTweet', 'idCandidato', 'timeStamp'],[hastag, '0', candidato, 0])
print(">>> new hastag saved for mining use 'main.py mineHashtag {} {} {}'".format(candidato, hastag, 'p'))
elif s1 == 'mineHashtag':
candidato, hastag = sys.argv[2], sys.argv[3]
p = False
if sys.argv[4] == 'p':
p = True
mining = HastagMining(hastag, candidato, print_status = p)
mining.start()
if s1 =='mineTweet':
arq = open('Last_id.txt','r')
display_name,logo,COMMUNITY_ID)
VALUES ('',
'{}',
'{}',
'',
'{}',
'',
'{}',
'{}',
'{}',
'',
'{}',
'{}',
'{}',
'{}');'''.format(merchant['business_name'],
merchant['phone_number'],
merchant['address'], '',
merchant['lat'], merchant['long'],
merchant['sub_title'], param['cat'],
merchant['logo'], COMMUNITY_ID)
dao.execute_dmls(insertSql)
except:
pass
if __name__ == '__main__':
querySql = 'SELECT a.name ,a.COMMUNITY_ID ,b.LATITUDE,b.LONGITUDE FROM shengchan_20140815.communities a ,shengchan_20140815.community_poses b WHERE a.COMMUNITY_ID =b.COMMUNITY_ID AND a.AREA_ID <11 AND a.AREA_ID > 0 '
dao = Dao()
result = dao.execute_query(querySql)
for name, COMMUNITY_ID, LATITUDE, LONGITUDE in result:
getmerchants(LATITUDE, LONGITUDE, COMMUNITY_ID)
class CommunitiesListCrawler(BaseCrawler, threading.Thread):
global Dao
Dao = Dao()
def __init__(self, page_num):
# TO
threading.Thread.__init__(self, name=page_num)
super().__init__(self)
self.detail_info_urls = []
self.source_id = 30
self.min_page = page_num * 30 + 1
self.max_page = page_num * 30 + 31
self._base_url = "http://newhouse.hfhome.cn/"
self._community_detail = {
'url': '',
'name': '',
'location': '',
'area_name': '',
'description': '',
'latitude': '',
'longitude': ''
}
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
# 单个url
html = self.get_page_content_str(seed_url)
self.findEachBuilding(html)
def findEachBuilding(self, html):
doc = Pq(html)
tr_list = doc("table#GVFwxkz>tr")
for tr in tr_list:
name = Pq(tr)("td:eq(1)").text()
self._community_detail["name"] = name
href = doc(tr).find("td>a").attr("href")
if href == None:
continue
href = href[href.index("?"):]
url = "http://newhouse.hfhome.cn/Modal/RoomList.aspx" + href
if self._check_community(url):
print(url + " --- 已经爬取过了")
continue
self._community_detail["url"] = url
self._extract_data(url)
def _extract_data(self, url):
community_id = self._save_community()
doc_str = self.get_page_content_str(url)
doc = Pq(doc_str)
tr_list = doc("table>tr")
try:
for tr in tr_list:
Floor_num = Pq(tr)("td:eq(0)").text()
a_list = doc(tr).find("td.preview>a")
for a in a_list:
apartment_detail = {
'COMMUNITY_ID': community_id,
'FLOOR_NUM': Floor_num,
'APARTMENT_NUM': doc(a).text(),
'STATUS': '2',
'create_time': time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
}
self._save_apartment(apartment_detail)
sql = "update communities set status = '2' where ORIGINAL_URL = '{}' ; ".format(url)
Dao.execute_dmls(sql)
except Exception as e:
print(e)
sql = "update communities set status = -1 where ORIGINAL_URL = '{}' ; ".format(url)
Dao.execute_dmls(sql)
def _insert_community(self):
result = "INSERT INTO communities (ORIGINAL_URL,NAME,AREA_NAME,LATITUDE,LONGITUDE,address,source_id )" \
" VALUES ('{}','{}','{}','{}','{}' ,'{}','{}' )".format(self._community_detail["url"],
self._community_detail["name"],
self._community_detail["area_name"],
self._community_detail["latitude"],
self._community_detail["longitude"],
self._community_detail["location"],
self.source_id)
return result
def _insert_apartment(self, apartment_detail):
result = "INSERT INTO apartments (COMMUNITY_ID , APARTMENT_NUM ,STATUS ,FLOOR_NUM,create_time )" \
" VALUES ('{}','{}','{}','{}','{}' )".format(apartment_detail["COMMUNITY_ID"],
apartment_detail["APARTMENT_NUM"],
apartment_detail["STATUS"],
apartment_detail["FLOOR_NUM"],
apartment_detail["create_time"])
return result
def _save_apartment(self, apartment_detail):
# 表中是否已有记录
query_sql = "SELECT * FROM apartments WHERE COMMUNITY_ID = {} and FLOOR_NUM ='{}' and APARTMENT_NUM ='{}' ".format(
int(apartment_detail["COMMUNITY_ID"]), apartment_detail["FLOOR_NUM"],
apartment_detail["APARTMENT_NUM"])
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(str(apartment_detail["COMMUNITY_ID"]) +
apartment_detail["FLOOR_NUM"] +
apartment_detail["APARTMENT_NUM"]))
return
# 数据插入操作
try:
Dao.execute_dmls(self._insert_apartment(apartment_detail))
except Exception as e:
print(e)
def _save_community(self):
# 表中是否已有记录
query_sql = "SELECT * FROM communities WHERE ORIGINAL_URL = '{}' and source_id ={} ".format(
self._community_detail["url"], self.source_id)
communityid_sql = "SELECT COMMUNITY_ID FROM communities WHERE ORIGINAL_URL = '{}' and source_id ={} ".format(
self._community_detail["url"], self.source_id)
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(self._community_detail["name"]))
return Dao.execute_query(communityid_sql)[0][0]
# 数据插入操作
Dao.execute_dmls(self._insert_community())
return Dao.execute_query(communityid_sql)[0][0]
def _check_community(self, url):
# 表中是否已有记录 完成的
communityid_sql = "SELECT COMMUNITY_ID FROM communities WHERE ORIGINAL_URL = '{}' and source_id ={} and status = 2 ".format(
url, self.source_id)
result = Dao.execute_query(communityid_sql)
if result == None:
return False
return True
def run(self):
# for i in range(self.min_page, self.max_page):
for i in range(363, 397):
url = "http://newhouse.hfhome.cn/hffd_xkz.aspx?page={}".format(i)
self._visit_pages(url)
def __init__(self):
self._conn = sqlite3.connect('database.db')
self.vaccines = Dao(DTO.Vaccine, self._conn)
self.suppliers = Dao(DTO.Supplier, self._conn)
self.clinics = Dao(DTO.Clinic, self._conn)
self.logistics = Dao(DTO.Logistic, self._conn)
class CommunitiesListCrawler(BaseCrawler):
global Dao
Dao = Dao()
def __init__(self):
# TODO 用参数化和多线程来执行抓取
super().__init__()
self.detail_info_urls = []
self.source_id = 21
self._base_url = "http://www.njhouse.com.cn/spf/inf/"
self._root_url = "http://www.njhouse.com.cn/spf/inf/index.php"
self._apartment_detail = {
'COMMUNITY_ID': 0,
'BUILDING_NUM': '',
'APARTMENT_NUM': '',
'STATUS': '2',
'create_time': ''
}
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
# for single_url in seed_url:
# update_sql = " UPDATE fetch_list SET times = times+1 WHERE url = '{}'and source_id =17".format(
# single_url[0])
# Dao.execute_dmls(update_sql)
# self._base_url = single_url[0]
# self._now_url = single_url[0]
# html = self.get_page_content_str(single_url[0])
# try:
# self._extract_data(html)
# except Exception as e:
# print(e)
# update_sql = " UPDATE fetch_list SET status = 1 WHERE url = '{}'and source_id =17".format(
# single_url[0])
# Dao.execute_dmls(update_sql)
# 单个url
# html = self.get_page_content_str(self._seed_url[0]) #用数据库的时候
seed_url = self._root_url + seed_url[seed_url.rindex("?"):]
html = self.get_page_content_str(seed_url) #单个URL
self.findEachBuilding(html)
# b = set(self._resualt)
# self._resualt=[i for i in b]
# # dao=Dao()
# insert_sql=""
# for res1 in b :
# insert_sql = "INSERT INTO merchant_tmp (description,url )VALUES ( '{}', 'http://www.youlin.me/category/407')".format(res1)
# print( insert_sql )
# dao = Dao()
# dao.execute_dmls(insert_sql)
def get_page_content_str(self, url):
try:
print("现在开始抓取" + url)
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'
}
request = urllib.request.Request(url=url, headers=headers)
m_fp = urllib.request.urlopen(request, timeout=1500)
html_str_uncode = m_fp.read()
if html_str_uncode == '':
print("出问题了，没出来数据")
return self.get_page_content_str(url)
m_fp.close()
return html_str_uncode
except urllib.error.URLError as err:
return None
except Exception as err:
print(err)
return None
def _generate_seed_url(self):
"""
generate all url to visit
"""
# self._seed_url = "http://www.njhouse.com.cn/spf/inf/index.php?prjid=108510"
# self._visit_pages(self._seed_url)
# from page 1 to anypage which < 200
# # 从数据库添加
# self._seed_url = Dao._get_url_by_id(self.source_id)
querysql = "SELECT COMMUNITY_ID,ORIGINAL_URL FROM communities WHERE source_id ='{}' and status= -1 ; ".format(
self.source_id)
result = Dao.execute_query(querysql)
for COMMUNITY_ID, ORIGINAL_URL in result:
try:
self._apartment_detail["COMMUNITY_ID"] = int(COMMUNITY_ID)
self._apartment_detail["create_time"] = time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
self._visit_pages(ORIGINAL_URL)
sql = "update communities set status = '2' where COMMUNITY_ID = '{}' ".format(
int(COMMUNITY_ID))
Dao.execute_dmls(sql)
except Exception as e:
print(e)
sql = "update communities set status = '-1' where COMMUNITY_ID = '{}' ".format(
int(COMMUNITY_ID))
Dao.execute_dmls(sql)
# 直接加，测试
# self._seed_url.append(self._base_url)
def findEachBuilding(self, html):
doc = Pq(html)
a_list = doc("table>tr>td.text>a")
for a in a_list:
self._apartment_detail["BUILDING_NUM"] = doc(a).text()
url = self._base_url + doc(a).attr("href")
doc_str = self.get_page_content_str(url)
self._extract_data(doc_str)
time.sleep(5)
def _extract_data(self, doc_str):
try:
doc = Pq(doc_str)
a_list = doc("tr.text>td>a")
# total_item =int( doc("").text().strip())
# count_num = int(total_item) / 12
for a in a_list:
self._apartment_detail["APARTMENT_NUM"] = doc(a).text()
if self._apartment_detail["APARTMENT_NUM"].strip() != '':
self._apartment_detail["create_time"] = time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
self._save_community()
except Exception as err:
print(err)
time.sleep(100)
self._extract_data(doc_str)
def _insert_community(self):
result = "INSERT INTO apartments (COMMUNITY_ID , BUILDING_NUM , APARTMENT_NUM ,STATUS ,create_time )" \
" VALUES ('{}','{}','{}','{}','{}' )".format(self._apartment_detail["COMMUNITY_ID"],
self._apartment_detail["BUILDING_NUM"],
self._apartment_detail["APARTMENT_NUM"],
self._apartment_detail["STATUS"],
self._apartment_detail["create_time"])
return result
def _save_community(self):
# 表中是否已有记录
query_sql = "SELECT * FROM apartments WHERE COMMUNITY_ID = {} and BUILDING_NUM ='{}' and APARTMENT_NUM ='{}' ".format(
int(self._apartment_detail["COMMUNITY_ID"]),
self._apartment_detail["BUILDING_NUM"],
self._apartment_detail["APARTMENT_NUM"])
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(
str(self._apartment_detail["COMMUNITY_ID"]) +
self._apartment_detail["BUILDING_NUM"] +
self._apartment_detail["APARTMENT_NUM"]))
return
# 数据插入操作
try:
Dao.execute_dmls(self._insert_community())
except Exception as e:
print(e)
def craw(self):
self._generate_seed_url()
class CommunitiesListCrawler(BaseCrawler):
global Dao
Dao = Dao()
def craw(self):
self._generate_seed_url()
def __init__(self):
# TODO 用参数化和多线程来执行抓取
super().__init__()
self.detail_info_urls = []
self.source_id = 18
self._base_url = "http://bj.5i5j.com"
self._community_detail = {
'url': '',
'name': '',
'location': '',
'area_name': '',
'description': '',
'latitude': '',
'longitude': '',
'city': ''
}
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
# for single_url in seed_url:
# update_sql = " UPDATE fetch_list SET times = times+1 WHERE url = '{}'and source_id =17".format(
# single_url[0])
# Dao.execute_dmls(update_sql)
# self._base_url = single_url[0]
# self._now_url = single_url[0]
# html = self.get_page_content_str(single_url[0])
# try:
# self._extract_data(html)
# except Exception as e:
# print(e)
# update_sql = " UPDATE fetch_list SET status = 1 WHERE url = '{}'and source_id =17".format(
# single_url[0])
# Dao.execute_dmls(update_sql)
# 单个url
html = self.get_page_content_str(seed_url)
self.findEachArea(html)
# b = set(self._resualt)
# self._resualt=[i for i in b]
# # dao=Dao()
# insert_sql=""
# for res1 in b :
# insert_sql = "INSERT INTO merchant_tmp (description,url )VALUES ( '{}', 'http://www.youlin.me/category/407')".format(res1)
# print( insert_sql )
# dao = Dao()
# dao.execute_dmls(insert_sql)
def _generate_seed_url(self):
"""
generate all url to visit
"""
# from page 1 to anypage which < 200
# # 从数据库添加
# self._seed_url = Dao._get_url_by_id(self.source_id)
# 直接加，测试
self._visit_pages("http://www.llzg.cn/nhd/neig44000000.html")
def findEachArea(self, HTML):
doc = Pq(HTML)
citys = doc(
"body > div > div.container > div > div > div.branchNavWrap")
for city in citys:
self._community_detail['city'] = doc(city).find(
"div.title3>h4").text()
areas = doc(city).find("div.branchNav>a")
for area in areas:
self._community_detail['area_name'] = doc(area).text()
url = doc(area).attr('href') + '/plot/plotlist'
self._extract_data(url)
def _extract_data(self, url):
doc_str = self.get_page_content_str(url)
doc = Pq(doc_str)
communities = doc(
"body > div.container.marginTop10px > div > div.span9 > div.areasWrap > div.areas > dl.clearfix > dd > a"
)
for community in communities:
self._community_detail['name'] = doc(community).text()
self._save_community()
def _insert_community(self):
result = "INSERT INTO ehdc.communities_llzg (ORIGINAL_URL,NAME,AREA_NAME,LATITUDE,LONGITUDE,city_name )" \
" VALUES ('{}','{}','{}','{}','{}' ,'{}' )".format(self._community_detail["url"],
self._community_detail["name"],
self._community_detail["area_name"],
'',
'',
self._community_detail['city'])
return result
def _save_community(self):
# 表中是否已有记录
query_sql = "SELECT * FROM ehdc.communities_llzg WHERE NAME = '{}' and AREA_NAME='{}' ".format(
self._community_detail["name"],
self._community_detail["area_name"])
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(
self._community_detail["name"]))
return
# 数据插入操作
Dao.execute_dmls(self._insert_community())
class CommunitiesListCrawler(BaseCrawler):
global Dao
Dao = Dao()
def __init__(self):
# TODO 用参数化和多线程来执行抓取
super().__init__()
self.detail_info_urls = []
self.source_id = 21
self._base_url = "http://newhouse.cnnbfdc.com/lpxx.aspx"
self._community_detail = {
'url': '',
'name': '',
'location': '',
'area_name': '',
'description': '',
'latitude': '',
'longitude': ''
}
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
# 单个url
html = self.get_page_content_str(seed_url)
self._extract_data(html)
# def get_page_content_str(self, url):
# time.sleep(1)
#
# try:
# print("现在开始抓取" + url)
# headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}
# request = urllib.request.Request(url=url, headers=headers)
# m_fp = urllib.request.urlopen(request, timeout=5500)
# html_str_uncode = m_fp.read()
# m_fp.close()
# return html_str_uncode
# except urllib.error.URLError as err:
# # logfile = open('test.log', 'a')
# # logfile.write("Error: {} \n in url : {}".format(err, url))
# # logfile.close()
# # print("error in {}.get_page_content_str".format(__name__))
# # if url[-3:] == "htm":
# # time.sleep(120)
# # return self.get_page_content_str(url)
# return None
# except Exception as err:
# print(err)
# return None
def _generate_seed_url(self):
"""
generate all url to visit
"""
# from page 1 to anypage which < 200
# # 从数据库添加
# self._seed_url = Dao._get_url_by_id(self.source_id)
# 直接加，测试
# self._visit_pages(self._base_url)
# 实际操作，循环每一页
for i in range(1, 70):
visiturl = self._base_url + '?p=' + str(i)
self._visit_pages(visiturl)
def _extract_data(self, doc_str):
doc = Pq(doc_str)
tr_list = doc("td.sp_sck>table>tr")
# total_item =int( doc("").text().strip())
# count_num = int(total_item) / 12
for tr in tr_list:
try:
doc = Pq(tr)
# test = doc(doc("tr")[1]).find("td")[1].text()
self._community_detail['location'] = doc("td:eq(3)").text()
self._community_detail['name'] = doc("a.sp_zi12c").text()
self._community_detail['url'] = doc("a.sp_zi12c").attr("href")
self._community_detail['area_name'] = doc("span.sp_f12").text()
if (self._community_detail['name'] != ''):
self._save_community()
except Exception as err:
print(tr)
print(err)
continue
def _insert_community(self):
result = "INSERT INTO communities (ORIGINAL_URL,NAME,AREA_NAME,LATITUDE,LONGITUDE,address,source_id )" \
" VALUES ('{}','{}','{}','{}','{}' ,'{}','{}' )".format(self._community_detail["url"],
self._community_detail["name"],
self._community_detail["area_name"],
self._community_detail["latitude"],
self._community_detail["longitude"],
self._community_detail["location"],
self.source_id)
return result
def _save_community(self):
# 表中是否已有记录
query_sql = "SELECT * FROM communities WHERE ORIGINAL_URL = '{}' and source_id ={} ".format(
self._community_detail["url"], self.source_id)
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(
self._community_detail["name"]))
return
# 数据插入操作
Dao.execute_dmls(self._insert_community())
def craw(self):
self._generate_seed_url()
class CommunitiesListCrawler(BaseCrawler):
global Dao
Dao = Dao()
def __init__(self):
# TODO 用参数化和多线程来执行抓取
super().__init__()
self.detail_info_urls = []
self.source_id = 18
self._base_url = "http://bj.5i5j.com"
self._community_detail = {
'url': '',
'name': '',
'location': '',
'area_name': '',
'description': '',
'latitude': '',
'longitude': ''
}
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
# for single_url in seed_url:
# update_sql = " UPDATE fetch_list SET times = times+1 WHERE url = '{}'and source_id =17".format(
# single_url[0])
# Dao.execute_dmls(update_sql)
# self._base_url = single_url[0]
# self._now_url = single_url[0]
# html = self.get_page_content_str(single_url[0])
# try:
# self._extract_data(html)
# except Exception as e:
# print(e)
# update_sql = " UPDATE fetch_list SET status = 1 WHERE url = '{}'and source_id =17".format(
# single_url[0])
# Dao.execute_dmls(update_sql)
# 单个url
html = self.get_page_content_str(self._seed_url[0])
self.findEachArea(html)
# b = set(self._resualt)
# self._resualt=[i for i in b]
# # dao=Dao()
# insert_sql=""
# for res1 in b :
# insert_sql = "INSERT INTO merchant_tmp (description,url )VALUES ( '{}', 'http://www.youlin.me/category/407')".format(res1)
# print( insert_sql )
# dao = Dao()
# dao.execute_dmls(insert_sql)
def _generate_seed_url(self):
"""
generate all url to visit
"""
# from page 1 to anypage which < 200
# # 从数据库添加
# self._seed_url = Dao._get_url_by_id(self.source_id)
# 直接加，测试
self._seed_url.append("http://bj.5i5j.com/community/")
def findEachArea(self,HTML):
doc = Pq(HTML)
li= doc(".c-info>.s-term2>li")[0]
aList = doc(li).find("a")
hrefList=[]
for a in aList :
a = doc(a).attr("href")
if a != "/community/" and a!="/community/chaoyang/" and a != "/community/haidian/" and a != "/community/fengtai/" and a!= "/community/dongcheng/" :
hrefList.append(self._base_url + a)
for href in hrefList :
html = self.get_page_content_str(href)
doc1 = Pq(html)
aList1 = doc1(".c-info>.s-term2>li>.pop1>ul>.text2>a")
for a in aList1 :
a = doc(a).attr("href")
self._extract_data(self._base_url + a)
def _extract_data(self, url):
doc_str = self.get_page_content_str(url)
if "抱歉，没有找到符合您要求的小区" in doc_str :
return False
doc = Pq(doc_str)
total_item =int( doc(".main-l>.reorder>b>font").text().strip())
count_num = int(total_item) / 12
for page in range(1, int(count_num) + 2):
url1 = url + "n" + str(page)
html = self.get_page_content_str(url1)
self._extract_data2(html)
def _extract_data2(self, doc_str):
doc = Pq(doc_str)
li_list = doc(".main-l>ul[class^='house-list']>li")
for li in li_list:
self._community_detail['latitude'] = doc(li).attr("y")
self._community_detail['longitude'] =doc(li).attr("x")
self._community_detail["url"] = self._base_url + doc(li).find("dl>dd>.xqbt>a").attr("href")
self._community_detail["name"] = doc(li).find("dl>dd>.xqbt>a").text()
self._community_detail["area_name"] = "".join(doc(li).find("dl>dd>.pw1015>span[class^='add']>a").text())
print(self._community_detail)
# url = doc(li).find(".details>.p_links>a").attr("href")
self._save_community()
def _insert_community(self):
result = "INSERT INTO communities (ORIGINAL_URL,NAME,AREA_NAME,LATITUDE,LONGITUDE,source_id )" \
" VALUES ('{}','{}','{}','{}','{}' ,'{}' )".format(self._community_detail["url"],
self._community_detail["name"],
self._community_detail["area_name"],
self._community_detail["latitude"],
self._community_detail["longitude"],
self.source_id)
return result
def _save_community(self):
# 表中是否已有记录
query_sql = "SELECT * FROM communities WHERE ORIGINAL_URL = '{}' and source_id = 18 ".format(self._community_detail["url"])
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(self._community_detail["name"]))
return
# 数据插入操作
Dao.execute_dmls(self._insert_community())
class CommunitiesListCrawler(BaseCrawler):
global Dao
Dao = Dao()
def __init__(self):
# TODO 用参数化和多线程来执行抓取
super().__init__()
self.detail_info_urls = []
self.source_id = 21
self._base_url = "http://www.njhouse.com.cn/persalereg.php"
self._community_detail = {
'url': '',
'name': '',
'location': '',
'area_name': '',
'description': '',
'latitude': '',
'longitude': ''
}
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
# for single_url in seed_url:
# update_sql = " UPDATE fetch_list SET times = times+1 WHERE url = '{}'and source_id =17".format(
# single_url[0])
# Dao.execute_dmls(update_sql)
# self._base_url = single_url[0]
# self._now_url = single_url[0]
# html = self.get_page_content_str(single_url[0])
# try:
# self._extract_data(html)
# except Exception as e:
# print(e)
# update_sql = " UPDATE fetch_list SET status = 1 WHERE url = '{}'and source_id =17".format(
# single_url[0])
# Dao.execute_dmls(update_sql)
# 单个url
html = self.get_page_content_str(self._seed_url[0])
self._extract_data(html)
# b = set(self._resualt)
# self._resualt=[i for i in b]
# # dao=Dao()
# insert_sql=""
# for res1 in b :
# insert_sql = "INSERT INTO merchant_tmp (description,url )VALUES ( '{}', 'http://www.youlin.me/category/407')".format(res1)
# print( insert_sql )
# dao = Dao()
# dao.execute_dmls(insert_sql)
def get_page_content_str(self, url):
time.sleep(1)
try:
print("现在开始抓取" + url)
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'
}
request = urllib.request.Request(url=url, headers=headers)
m_fp = urllib.request.urlopen(request, timeout=5500)
html_str_uncode = m_fp.read()
m_fp.close()
return html_str_uncode
except urllib.error.URLError as err:
# logfile = open('test.log', 'a')
# logfile.write("Error: {} \n in url : {}".format(err, url))
# logfile.close()
# print("error in {}.get_page_content_str".format(__name__))
# if url[-3:] == "htm":
# time.sleep(120)
# return self.get_page_content_str(url)
return None
except Exception as err:
print(err)
return None
def _generate_seed_url(self):
"""
generate all url to visit
"""
# from page 1 to anypage which < 200
# # 从数据库添加
# self._seed_url = Dao._get_url_by_id(self.source_id)
# 直接加，测试
self._seed_url.append(self._base_url)
def _extract_data(self, doc_str):
doc = Pq(doc_str)
tables = doc("table>tr>td>table")
# total_item =int( doc("").text().strip())
# count_num = int(total_item) / 12
for table in tables:
try:
doc = Pq(table)
# test = doc(doc("tr")[1]).find("td")[1].text()
self._community_detail['location'] = Pq(
doc("tr:eq(1)"))("td:eq(1)").text()
self._community_detail['name'] = Pq(
doc("tr:eq(2)"))("a").text()
self._community_detail['url'] = Pq(
doc("tr:eq(2)"))("a").attr("href")
self._community_detail['area_name'] = Pq(
doc("tr:eq(8)"))("td:eq(1)").text()
self._save_community()
except Exception as err:
print(table)
print(err)
continue
def _insert_community(self):
result = "INSERT INTO communities (ORIGINAL_URL,NAME,AREA_NAME,LATITUDE,LONGITUDE,location,source_id )" \
" VALUES ('{}','{}','{}','{}','{}' ,'{}','{}' )".format(self._community_detail["url"],
self._community_detail["name"],
self._community_detail["area_name"],
self._community_detail["latitude"],
self._community_detail["longitude"],
self._community_detail["location"],
self.source_id)
return result
def _save_community(self):
# 表中是否已有记录
query_sql = "SELECT * FROM communities WHERE ORIGINAL_URL = '{}' and source_id ={} ".format(
self._community_detail["url"], self.source_id)
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(
self._community_detail["name"]))
return
# 数据插入操作
Dao.execute_dmls(self._insert_community())
class CommunitiesListCrawler(BaseCrawler):
global Dao
Dao = Dao()
def __init__(self):
# TODO 用参数化和多线程来执行抓取
super().__init__()
self.detail_info_urls = []
self.source_id = 17
self._base_url = "http://sz.ganji.com/"
self._community_detail = {
'url': '',
'name': '',
'location': '',
'area_name': '',
'description': '',
'latitude': '',
'longitude': ''
}
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
for single_url in seed_url:
update_sql = " UPDATE fetch_list SET times = times+1 WHERE url = '{}'and source_id =17".format(
single_url[0])
Dao.execute_dmls(update_sql)
self._base_url = single_url[0]
self._now_url = single_url[0]
html = self.get_page_content_str(single_url[0])
try:
self._extract_data(html)
except Exception as e:
print(e)
update_sql = " UPDATE fetch_list SET status = 1 WHERE url = '{}'and source_id =17".format(
single_url[0])
Dao.execute_dmls(update_sql)
# 单个url
# html = self.get_page_content_str(self._seed_url[0])
# self._extract_data(html)
# b = set(self._resualt)
# self._resualt=[i for i in b]
# # dao=Dao()
# insert_sql=""
# for res1 in b :
# insert_sql = "INSERT INTO merchant_tmp (description,url )VALUES ( '{}', 'http://www.youlin.me/category/407')".format(res1)
# print( insert_sql )
# dao = Dao()
# dao.execute_dmls(insert_sql)
def _generate_seed_url(self):
"""
generate all url to visit
"""
# from page 1 to anypage which < 200
# 从数据库添加
self._seed_url = Dao._get_url_by_id(self.source_id)
# 直接加，测试
# self._seed_url.append("http://www.anjuke.com/index.html")
def _extract_data(self, doc_str):
doc = Pq(doc_str)
total_item = doc(".PL_1.clearfix>span>em").text()
count_num = int(total_item) / 10
for page in range(1, int(count_num) + 1):
url = self._base_url + "W0QQpZ" + str(page)
html = self.get_page_content_str(url)
self._extract_data2(html)
def _extract_data2(self, doc_str):
doc = Pq(doc_str)
li_list = doc(".mainListing.clearfix>.pL>.list>li")
for li in li_list:
self._community_detail["url"] = doc(li).find(
".details>div>a").attr("href")
self._community_detail["name"] = doc(li).find(
".details>div>a").text()
p = doc(li).find(".details>p")
self._community_detail["location"] = doc(p[0]).text()
self._community_detail["area_name"] = self._community_detail[
"location"][self._community_detail["location"].index("[") +
1:self._community_detail["location"].index("]")]
self._community_detail["location"] = self._community_detail[
"location"][self._community_detail["location"].index("]") + 1:]
url = doc(li).find(".details>.p_links>a").attr("href")
self._community_detail['latitude'] = url[url.index("l1=") +
3:url.index("&l2")]
self._community_detail['longitude'] = url[url.index("l2=") +
3:url.index("&l3")]
self._save_community()
def _insert_community(self):
result = "INSERT INTO communities (ORIGINAL_URL,NAME,address,AREA_NAME,LATITUDE,LONGITUDE,source_id )" \
" VALUES ('{}','{}','{}','{}','{}','{}' ,'{}' )".format(self._community_detail["url"],
self._community_detail["name"],
self._community_detail["location"],
self._community_detail["area_name"],
self._community_detail["latitude"],
self._community_detail["longitude"],
self.source_id)
return result
def _save_community(self):
# 表中是否已有记录
query_sql = "SELECT * FROM communities WHERE ORIGINAL_URL = '{}'".format(
self._community_detail["url"])
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(
self._community_detail["name"]))
return
# 数据插入操作
Dao.execute_dmls(self._insert_community())
class CommunitiesListCrawler(BaseCrawler):
global Dao
Dao = Dao()
def __init__(self):
# TODO 用参数化和多线程来执行抓取
super().__init__()
self.detail_info_urls = []
self.source_id = 28
self._base_url = "http://newhouse.cnnbfdc.com/GetHouseTable.aspx"
self._apartment_detail = {
'COMMUNITY_ID': 0,
'BUILDING_NUM': '',
'APARTMENT_NUM': '',
'STATUS': '2',
'create_time': ''
}
def _visit_pages(self, seed_url, apartment_detail):
"""
visit one url,get page content
"""
# 单个url
# html = self.get_page_content_str(self._seed_url[0]) #用数据库的时候
endurl = seed_url[seed_url.index("?"):seed_url.index("&projectid")]
seed_url = self._base_url + endurl
html = self.get_page_content_str(seed_url) # 单个URL
self._extract_data(html, apartment_detail)
# b = set(self._resualt)
# self._resualt=[i for i in b]
# # dao=Dao()
# insert_sql=""
# for res1 in b :
# insert_sql = "INSERT INTO merchant_tmp (description,url )VALUES ( '{}', 'http://www.youlin.me/category/407')".format(res1)
# print( insert_sql )
# dao = Dao()
# dao.execute_dmls(insert_sql)
def _generate_seed_url(self):
"""
generate all url to visit
"""
# self._seed_url = "http://www.njhouse.com.cn/spf/inf/index.php?prjid=108510"
# self._visit_pages(self._seed_url)
# from page 1 to anypage which < 200
# # 从数据库添加
# self._seed_url = Dao._get_url_by_id(self.source_id)
querysql = "SELECT COMMUNITY_ID, BUILDING_NUM, URL FROM ehdc.buildings WHERE STATUS = 0 ; "
result = Dao.execute_query(querysql)
for COMMUNITY_ID, BUILDING_NUM, URL in result:
self.execute(COMMUNITY_ID, BUILDING_NUM, URL)
def execute(self, COMMUNITY_ID, BUILDING_NUM, URL):
try:
apartment_detail = {
'COMMUNITY_ID': 0,
'BUILDING_NUM': '',
'APARTMENT_NUM': '',
'STATUS': '2',
'create_time': ''
}
apartment_detail["COMMUNITY_ID"] = int(COMMUNITY_ID)
apartment_detail["BUILDING_NUM"] = BUILDING_NUM
apartment_detail["create_time"] = time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
self._visit_pages(URL, apartment_detail)
sql = "update BUILDINGS set status = '2' where URL = '{}' ; ".format(
URL)
Dao.execute_dmls(sql)
except Exception as e:
print(e)
sql = "update BUILDINGS set status = -1 where URL = '{}' ; ".format(
URL)
Dao.execute_dmls(sql)
def findEachBuilding(self, html):
doc = Pq(html)
a_list = doc("a.e_huangse")
for a in a_list:
self._apartment_detail["BUILDING_NUM"] = doc(a).text()
href = doc(a).attr("onclick")
href = href[href.index("'") + 1:]
href = href[:href.index("'")]
url = self._base_url + href
# doc_str = self.get_page_content_str(url)
# elf._extract_data(doc_str)
# time.sleep(1)
self.save_building(url)
def _extract_data(self, doc_str, apartment_detail):
try:
doc = Pq(doc_str)
a_list = doc("table>tr>td>table>tr>td")
# total_item =int( doc("").text().strip())
# count_num = int(total_item) / 12
for a in a_list:
apartment_detail["APARTMENT_NUM"] = doc(a).text()
if apartment_detail["APARTMENT_NUM"].strip() != '':
apartment_detail["create_time"] = time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
self._save_community(apartment_detail)
except Exception as err:
print(err)
time.sleep(1)
self._extract_data(doc_str)
def _insert_apartment(self, apartment_detail):
result = "INSERT INTO apartments (COMMUNITY_ID , BUILDING_NUM , APARTMENT_NUM ,STATUS ,create_time )" \
" VALUES ('{}','{}','{}','{}','{}' )".format(apartment_detail["COMMUNITY_ID"],
apartment_detail["BUILDING_NUM"],
apartment_detail["APARTMENT_NUM"],
apartment_detail["STATUS"],
apartment_detail["create_time"])
return result
def _save_community(self, apartment_detail):
# 表中是否已有记录
query_sql = "SELECT * FROM apartments WHERE COMMUNITY_ID = {} and BUILDING_NUM ='{}' and APARTMENT_NUM ='{}' ".format(
int(apartment_detail["COMMUNITY_ID"]),
apartment_detail["BUILDING_NUM"],
apartment_detail["APARTMENT_NUM"])
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(
str(apartment_detail["COMMUNITY_ID"]) +
apartment_detail["BUILDING_NUM"] +
apartment_detail["APARTMENT_NUM"]))
return
# 数据插入操作
try:
Dao.execute_dmls(self._insert_apartment(apartment_detail))
except Exception as e:
print(e)
def craw(self):
self._generate_seed_url()
def save_building(self, url):
SQL = " INSERT INTO ehdc.buildings (COMMUNITY_ID,BUILDING_NUM,URL,STATUS) VALUES ('{}','{}','{}','{}') ;".format(
str(self._apartment_detail["COMMUNITY_ID"]),
self._apartment_detail["BUILDING_NUM"], url,
self._apartment_detail["STATUS"])
Dao.execute_dmls(SQL)
class CategoryListCrawler(BaseCrawler):
global Dao
Dao = Dao()
def __init__(self):
super().__init__()
self.detail_info_urls = []
self.source_id = 17
self._base_url = "http://www.dianping.com/wuhan"
self._category_detail = {'shopType': '', 'categoryId': '', 'name': ''}
self._category_list = []
def get_page_content_str(self, url):
print("现在开始抓取" + url)
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'
}
request = urllib.request.Request(url=url, headers=headers)
m_fp = urllib.request.urlopen(request, timeout=500)
html_str = m_fp.read().decode('utf-8')
m_fp.close()
return html_str
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
# 单个url
html = self.get_page_content_str(self._seed_url[0])
self._extract_data(html)
def _generate_seed_url(self):
"""
generate all url to visit
"""
# 直接加，测试
self._seed_url.append(self._base_url)
def _extract_data(self, doc_str):
doc = Pq(doc_str)
li_list = doc('.aside.aside-left>.category-nav.J-category-nav>li')
for li in li_list:
self._category_detail["shopType"] = doc(li).attr("data-key")
self._category_detail["categoryId"] = self._category_detail[
"shopType"]
self._category_detail["name"] = doc(li).find(".name>span").text()
self._category_list.append(copy.copy(self._category_detail))
# doc2 = Pq(doc_str)
# div_list = doc2(".aside.aside-left>.category-nav.J-category-nav>li>.secondary-category.J-secondary-category>div>div")
a_list = doc(li).find("div>a")
for a in a_list:
self._category_detail["categoryId"] = doc(a).attr("data-key")
self._category_detail["name"] = doc(a).text()
self._category_list.append(copy.copy(self._category_detail))
self.save_category()
def craw(self):
self._generate_seed_url()
self._visit_pages(self._seed_url)
print(self._seed_url)
def save_category(self):
for category in self._category_list:
sql = "INSERT INTO category (shopType,categoryId,name ) VALUES ('{}','{}','{}' )".format(
category["shopType"], category["categoryId"], category["name"])
Dao.execute_dmls(sql)
class CommunitiesListCrawler(BaseCrawler, threading.Thread):
global Dao
Dao = Dao()
def __init__(self):
super().__init__()
self.detail_info_urls = []
self.source_id = 31
self._base_url = "http://www.tywsfdc.com/"
self._root_url = "http://www.tywsfdc.com/Firsthand/tyfc/publish/p/ProNBList.do?pid"
self._apartment_detail = {
'COMMUNITY_ID': 0,
'BUILDING_NUM': '',
'APARTMENT_NUM': '',
'STATUS': '2',
'create_time': ''
}
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
# 单个url
# html = self.get_page_content_str(self._seed_url[0]) #用数据库的时候
self._pid = seed_url[seed_url.rindex("-"):]
seed_url = self._root_url + "=" + self._pid
# print("_visit_pages " + seed_url)
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0',
'Referer': seed_url
}
values = {'pid': self._pid, 'pageNo': '1', 'pageSize': '50'}
data = urllib.parse.urlencode(values).encode(encoding='UTF8')
request = urllib.request.Request(
url="http://www.tywsfdc.com/Firsthand/tyfc/publish/ProNBList.do",
headers=headers,
data=data)
m_fp = urllib.request.urlopen(request, timeout=500)
html_str = m_fp.read().decode("utf8")
self.findEachBuilding(html_str)
# b = set(self._resualt)
# self._resualt=[i for i in b]
# # dao=Dao()
# insert_sql=""
# for res1 in b :
# insert_sql = "INSERT INTO merchant_tmp (description,url )VALUES ( '{}', 'http://www.youlin.me/category/407')".format(res1)
# print( insert_sql )
# dao = Dao()
# dao.execute_dmls(insert_sql)
def get_page_content_str(self, url):
try:
print("现在开始抓取" + url)
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'
}
request = urllib.request.Request(url=url, headers=headers)
m_fp = urllib.request.urlopen(request, timeout=1500)
html_str_uncode = m_fp.read()
if html_str_uncode == '':
print("出问题了，没出来数据")
return self.get_page_content_str(url)
m_fp.close()
return html_str_uncode
except urllib.error.URLError as err:
return None
except Exception as err:
print(err)
return None
def _generate_seed_url(self):
"""
generate all url to visit
"""
# self._seed_url = "http://www.njhouse.com.cn/spf/inf/index.php?prjid=108510"
# self._visit_pages(self._seed_url)
# from page 1 to anypage which < 200
# # 从数据库添加
# self._seed_url = Dao._get_url_by_id(self.source_id)
querysql = "SELECT COMMUNITY_ID,ORIGINAL_URL FROM communities WHERE source_id ='{}' and status<2 ; ".format(
self.source_id)
result = Dao.execute_query(querysql)
for COMMUNITY_ID, ORIGINAL_URL in result:
try:
self._apartment_detail["COMMUNITY_ID"] = int(COMMUNITY_ID)
self._apartment_detail["create_time"] = time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
# print("_generate_seed_url func : "+ORIGINAL_URL)
self._visit_pages(ORIGINAL_URL)
sql = "update communities set status = '2' where COMMUNITY_ID = '{}' ".format(
int(COMMUNITY_ID))
Dao.execute_dmls(sql)
except Exception as e:
print(e)
sql = "update communities set status = '-1' where COMMUNITY_ID = '{}' ".format(
int(COMMUNITY_ID))
Dao.execute_dmls(sql)
# 直接加，测试
# self._seed_url.append(self._base_url)
def findEachBuilding(self, html):
doc = Pq(html)
tr_list = doc("table>tr")
# print("tr size ")
for tr in tr_list:
try:
# 进入每一栋的url
objid = doc(tr).attr("objid")
if objid == None:
continue
self._apartment_detail["BUILDING_NUM"] = Pq(tr)(
"td:eq(2)").text()
url = "http://www.tywsfdc.com/Firsthand/tyfc/publish/p/ProNBView.do?proPID={}&nbid={}".format(
self._pid, objid)
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0',
'Referer': url
}
RequestURL = "http://www.tywsfdc.com/Firsthand/tyfc/publish/probld/NBView.do?nid={}&projectid={}".format(
objid, self._pid)
values = {'nid': objid, 'projectid': self._pid}
data = urllib.parse.urlencode(values).encode(encoding='UTF8')
request = urllib.request.Request(url=RequestURL,
headers=headers,
data=data)
m_fp = urllib.request.urlopen(request, timeout=500)
html_str = m_fp.read().decode("utf8")
self._extract_data(html_str)
except Exception as e:
print(e)
pass
def _extract_data(self, doc_str):
try:
doc = Pq(doc_str)
# 每一单元
building_list = doc("ul#bldlist>span")
for building in building_list:
bld = doc(building).attr("id")
bld = bld[3:]
self._apartment_detail["BUILDING_NUM"] = doc(building).text()
# 每一层：
xpath = "div.flrlist>table#{}>tr".format(bld)
tr_list = doc(xpath)
# total_item =int( doc("").text().strip())
# count_num = int(total_item) / 12
for tr in tr_list:
self._apartment_detail["FLOOR_NUM"] = Pq(tr)(
"td:eq(0)").text()
a_list = Pq(tr)("td:eq(1)>span>a")
for a in a_list:
self._apartment_detail["APARTMENT_NUM"] = doc(a).text()
if self._apartment_detail["APARTMENT_NUM"].strip(
) != '':
self._apartment_detail[
"create_time"] = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
self._save_apartments()
except Exception as err:
print(err)
time.sleep(100)
self._extract_data(doc_str)
def _insert_community(self):
result = "INSERT INTO apartments (COMMUNITY_ID , BUILDING_NUM ,FLOOR_NUM , APARTMENT_NUM ,STATUS ,create_time )" \
" VALUES ('{}','{}','{}','{}','{}','{}' )".format(self._apartment_detail["COMMUNITY_ID"],
self._apartment_detail["BUILDING_NUM"],
self._apartment_detail["FLOOR_NUM"],
self._apartment_detail["APARTMENT_NUM"],
self._apartment_detail["STATUS"],
self._apartment_detail["create_time"])
return result
def _save_apartments(self):
# 表中是否已有记录
query_sql = "SELECT * FROM apartments WHERE COMMUNITY_ID = {} and BUILDING_NUM ='{}' and APARTMENT_NUM ='{}'and FLOOR_NUM='{}' ; ".format(
int(self._apartment_detail["COMMUNITY_ID"]),
self._apartment_detail["BUILDING_NUM"],
self._apartment_detail["APARTMENT_NUM"],
self._apartment_detail["FLOOR_NUM"])
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(
str(self._apartment_detail["COMMUNITY_ID"]) +
self._apartment_detail["BUILDING_NUM"] +
self._apartment_detail["APARTMENT_NUM"]))
return
# 数据插入操作
try:
Dao.execute_dmls(self._insert_community())
except Exception as e:
print(e)
def craw(self):
self._generate_seed_url()
class CommunitiesListCrawler(BaseCrawler):
global Dao
Dao = Dao()
def __init__(self):
# TODO 用参数化和多线程来执行抓取
super().__init__()
self.detail_info_urls = []
self.source_id = 22
self._base_url = "http://www.funi.com"
self._community_detail = {
'url': '',
'name': '',
'location': '',
'area_name': '',
'description': '',
'latitude': '',
'longitude': ''
}
def _visit_pages(self, seed_url):
"""
visit one url,get page content
"""
# for single_url in seed_url:
# update_sql = " UPDATE fetch_list SET times = times+1 WHERE url = '{}'and source_id =17".format(
# single_url[0])
# Dao.execute_dmls(update_sql)
# self._base_url = single_url[0]
# self._now_url = single_url[0]
# html = self.get_page_content_str(single_url[0])
# try:
# self._extract_data(html)
# except Exception as e:
# print(e)
# update_sql = " UPDATE fetch_list SET status = 1 WHERE url = '{}'and source_id =17".format(
# single_url[0])
# Dao.execute_dmls(update_sql)
# 单个url
html = self.get_page_content_str(self._seed_url[0])
self.findEachArea(html)
# b = set(self._resualt)
# self._resualt=[i for i in b]
# # dao=Dao()
# insert_sql=""
# for res1 in b :
# insert_sql = "INSERT INTO merchant_tmp (description,url )VALUES ( '{}', 'http://www.youlin.me/category/407')".format(res1)
# print( insert_sql )
# dao = Dao()
# dao.execute_dmls(insert_sql)
def _generate_seed_url(self):
"""
generate all url to visit
"""
# from page 1 to anypage which < 200
# # 从数据库添加
# self._seed_url = Dao._get_url_by_id(self.source_id)
# 直接加，测试
self._seed_url.append("http://www.funi.com/loupan/region_605_0_HOUSE_0_1")
def findEachArea(self, HTML):
doc = Pq(HTML)
aList = doc(".house-search>.s-con>dl>dd>a")
for a in aList:
self._community_detail["area_name"] = doc(a).text()
a = doc(a).attr("href")
if a != "/loupan/region_0_0_HOUSE_0_1" and a != "/community/chaoyang/" and a != "/community/haidian/" and a != "/community/fengtai/" and a != "/community/dongcheng/":
self._extract_data(self._base_url + a)
def _extract_data(self, url):
doc_str = self.get_page_content_str(url)
if "抱歉，没有找到符合您要求的小区" in doc_str:
return False
doc = Pq(doc_str)
total_item = int(doc(".fleft>.pan-tab>dl>dt>p>i").text().strip())
count_num = int(total_item) / 10
for page in range(1, int(count_num) + 2):
url1 = url[:url.rindex("_")] + "_" + str(page)
html = self.get_page_content_str(url1)
self._extract_data2(html)
def _extract_data2(self, doc_str):
doc = Pq(doc_str)
li_list = doc(".fleft>.pan-con.maplist.clearfix>dl")
for li in li_list:
self._community_detail['location'] = doc(li).find("i.address").attr("title")
self._community_detail["url"] = self._base_url + doc(li).find(".clearfix>h2>a").attr("href")
self._community_detail["name"] = doc(li).find(".clearfix>h2>a").text()
print(self._community_detail)
# url = doc(li).find(".details>.p_links>a").attr("href")
self._save_community()
def _insert_community(self):
result = "INSERT INTO communities (ORIGINAL_URL,NAME,AREA_NAME,LATITUDE,LONGITUDE,location,source_id )" \
" VALUES ('{}','{}','{}','{}','{}' ,'{}','{}' )".format(self._community_detail["url"],
self._community_detail["name"],
self._community_detail["area_name"],
self._community_detail["latitude"],
self._community_detail["longitude"],
self._community_detail["location"],
self.source_id)
return result
def _save_community(self):
# 表中是否已有记录
query_sql = "SELECT * FROM communities WHERE ORIGINAL_URL = '{}' and source_id ={} ".format(
self._community_detail["url"], self.source_id)
if Dao.execute_query(query_sql) is not None:
print(" {} is already exists ,so next".format(self._community_detail["name"]))
return
# 数据插入操作
Dao.execute_dmls(self._insert_community())