Beispiel #1
0
    def get_build_info(self, build_id_list, co_id):
        bu = Building(co_index)
        for build_id in build_id_list:
            formdata = {}
            formdata["action"] = "qeurySingleBuilding"
            formdata['pk'] = str(build_id)
            header = {
                'User-Agent':
                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
                'Referer':
                'http://hkrealestate.haikou.gov.cn/wp_myself/housequery/projectBuildingList.php'
            }
            try:
                build_info = self.s.post(
                    'http://hkrealestate.haikou.gov.cn/wp_myself/housequery/projectBuildHouseAction.php',
                    data=formdata,
                    headers=header)
            except Exception as e:
                print("co_idnex={},楼栋错误".format(co_index), e)

            build_con = build_info.text
            bu.bu_id = build_id
            bu.co_id = co_id
            bu.bu_num = re.search('幢名称.*?<td>(.*?)<', build_con,
                                  re.S | re.M).group(1)
            bu.bu_floor = re.search('总层数.*?<td>(.*?)<', build_con,
                                    re.S | re.M).group(1)
            bu.bu_build_size = re.search('>建筑面积.*?<td>(.*?)<', build_con,
                                         re.S | re.M).group(1)
            bu.bo_develops = re.search('房地产企业.*?">(.*?)</td', build_con,
                                       re.S | re.M).group(1)

            bu.insert_db()

            self.get_house_info(build_con, co_id, build_id)
Beispiel #2
0
 def get_build_info(self, build_url_list):
     for i in build_url_list:
         try:
             build = Building(co_index)
             build_url = 'http://www.ndjsj.gov.cn/House/' + i
             build.co_name = '项目名称:.*?<td.*?>(.*?)<'
             build.bu_num = '幢  号:.*?<td.*?>(.*?)<'
             build.bu_address = '坐落位置:.*?<td.*?>(.*?)<'
             build.co_build_structural = '建筑结构:.*?<td.*?>(.*?)<'
             build.bu_floor = '总 层 数:.*?<td.*?>(.*?)<'
             build.bu_build_size = '总 面 积:.*?<td.*?>(.*?)<'
             # build.bu_type = '设计用途:.*?<td.*?>(.*?)<'
             build.bu_all_house = '批准销售:.*?<td.*?>(.*?)<'
             p = ProducerListUrl(
                 page_url=build_url,
                 request_type='get',
                 encode='utf-8',
                 analyzer_rules_dict=build.to_dict(),
                 current_url_rule='javascript:ShowTitle.*?href="(.*?)"',
                 analyzer_type='regex',
                 headers=self.headers)
             house_url_list = p.get_details()
             self.get_house_info(house_url_list)
         except Exception as e:
             print('宁德楼栋错误,url={}'.format(build_url), e)
Beispiel #3
0
 def get_comm_info(self, comm_url,comm):
     try:
         response = requests.get(comm_url, headers=self.headers)
         html = response.text
         comm.co_id = re.search('jectcode=(.*?)"', html, re.S | re.M).group(1)
         comm.co_name = re.search("项目名称:.*?<td.*?>(.*?)<", html, re.S | re.M).group(1)
         comm.co_address = re.search('项目地址:.*?<td.*?>(.*?)<', html, re.S | re.M).group(1)
         comm.co_develops = re.search('开发企业:.*?<td.*?>(.*?)<', html, re.S | re.M).group(1)
         comm.co_owner = re.search('国土证书:.*?<td.*?>(.*?)<', html, re.S | re.M).group(1)
         comm.area = re.search('行政区划:</th>.*?<td.*?>(.*?)<', html, re.S | re.M).group(1)
         comm.insert_db()
         build_html = re.search('套房信息.*?</table>', html, re.S | re.M).group()
         build_info_list = re.findall('<tr.*?>.*?</tr>', build_html, re.S | re.M)
         for i in build_info_list:
             try:
                 build = Building(co_index)
                 build.co_id = comm.co_id
                 build.bu_num = re.search('<td.*?>(.*?)</td', i, re.S | re.M).group(1)
                 build.bu_id = re.search('buildingcode=(.*?)&', i, re.S | re.M).group(1)
                 build.co_build_structural = re.search('<td.*?<td.*?<td.*?>(.*?)<', i, re.S | re.M).group(1)
                 build.bu_all_house = re.search('<td.*?<td.*?<td.*?<td.*?>(.*?)<', i, re.S | re.M).group(1)
                 build.bu_floor = re.search('<td.*?<td.*?<td.*?<td.*?<td.*?>(.*?)<', i, re.S | re.M).group(1)
                 build.insert_db()
                 house_url = re.search('href="(.*?)"', i, re.S | re.M).group(1)
                 self.get_build_info(house_url, build.bu_id, comm.co_id)
             except Exception as e:
                 print('楼栋错误,co_index={},url={}'.format(co_index, comm_url), e)
     except Exception as e:
         print('小区错误,co_index={},url={}'.format(co_index, comm_url), e)
Beispiel #4
0
 def bu_parse(self, bu_url, co_id, co_url):
     build_url = "http://61.143.241.154/" + bu_url
     global headers
     headers = {
         'User-Agent':
         'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119Safari/537.36',
         'Referer': co_url
     }
     bu_res = requests.get(build_url, headers=headers)
     bu_con = bu_res.content.decode('gbk')
     bu_pre_sale = re.search('预售许可证编号.*?blank">(.*?)</a', bu_con,
                             re.S | re.M).group(1)
     bu_pre_sale_date = re.search('预售证有效日期.*?">(.*?)</td', bu_con,
                                  re.S | re.M).group(1)
     bu_html = etree.HTML(bu_con)
     bu_list = bu_html.xpath("//table[@id='donglist']//tr")
     for bo in bu_list:
         bu = Building(co_index)
         bu.co_id = co_id
         bo_url = bo.xpath("./td/a/@href")[0]
         bu.bu_id = re.search('dbh=(.*?)&', bo_url).group(1)
         bu.bu_num = bo.xpath("./td[3]/text()")[0]
         bu.bu_floor = bo.xpath("./td[4]/text()")[0]
         bu.bu_pre_sale = bu_pre_sale
         bu.bu_pre_sale_date = bu_pre_sale_date
         bu.insert_db()
         self.house_parse(bo_url, co_id, bu.bu_id)
Beispiel #5
0
    def get_build_info(self, co_id, co_name):
        url = 'http://www.czhome.com.cn/Presell.asp?projectID=' + co_id + '&projectname=' + co_name
        response = requests.get(url, headers=self.headers)
        html = response.content.decode('gbk')
        tree = etree.HTML(html)
        xpath_list = tree.xpath('//tr[@class="indextabletxt"]')
        for i in xpath_list[1:]:
            build_url = i.xpath('td[2]/a/@href')[0]
            url = 'http://www.czhome.com.cn/' + build_url
            result = requests.get(url, headers=self.headers)
            if result.status_code is not 200:
                print("co_index={},预售url:{}连接失败".format(co_index, url))
                continue
            html = result.content.decode('gbk')
            tree = etree.HTML(html)
            # 总套数
            bu_xpath = tree.xpath(
                '/html/body/table/tr/td/table/tr/td/table/tr')[1:]
            for i in bu_xpath:
                try:
                    building = Building(7)
                    global building_id
                    building_id += 1
                    building.bu_id = building_id
                    bu_all_house = i.xpath('td[7]/text()')[0]
                    bu_url = i.xpath('td[1]/a/@href')[0]
                    url = 'http://www.czhome.com.cn/' + bu_url
                    response = requests.get(url, headers=self.headers)
                    if response.status_code is not 200:
                        print("co_index={},楼栋url:{}连接失败".format(co_index, url))
                        continue
                    html = response.content.decode('gbk')
                    tree = etree.HTML(html)
                    # 楼层
                    bu_floor = tree.xpath(
                        '//*[@id="Table4"]/tr[2]/td/table[3]/tr/td[1]/u/text()'
                    )[-1]
                    house_url_list = tree.xpath(
                        '//*[@id="Table4"]/tr[2]/td/table[3]/tr/td/a/@href')
                    bu_address = re.search(
                        '<center><font color=.*?&nbsp;&nbsp;(.*?)<', html,
                        re.S | re.M).group(1)
                    building.bu_all_house = bu_all_house
                    building.bu_address = bu_address
                    building.bu_floor = bu_floor
                    building.bu_id = building_id
                    building.co_id = co_id
                    building.insert_db()
                    for i in house_url_list:
                        try:
                            house = House(7)
                            house_url = 'http://www.czhome.com.cn/' + i
                            self.get_house_info(house_url, house, co_id,
                                                building_id, building)
                        except Exception as e:
                            print(e)

                except Exception as e:
                    print(e)
Beispiel #6
0
 def get_build_info(self, build_url_list, comm):
     for i in build_url_list:
         try:
             build_url = 'http://58.51.240.121:8503/' + i
             response = requests.get(build_url, headers=self.headers)
             html = response.text
             comm.co_pre_sale = re.search(
                 'id="PresellInfo1_lblXkzh">(.*?)<', html,
                 re.S | re.M).group(1)
             comm.co_pre_sale_date = re.search(
                 'id="PresellInfo1_lblFzrq">(.*?)<', html,
                 re.S | re.M).group(1)
             comm.insert_db()
             build_info_list = re.findall('<tr bgcolor="#FFFFFF">.*?</tr>',
                                          html, re.S | re.M)
             for i in build_info_list:
                 build = Building(co_index)
                 build.co_id = comm.co_id
                 build.bu_num = re.search('<td.*?>(.*?)<', i,
                                          re.S | re.M).group(1)
                 build.bu_floor = re.search('<td.*?<td.*?>(.*?)<', i,
                                            re.S | re.M).group(1)
                 build.bu_all_house = re.search('<td.*?<td.*?<td.*?>(.*?)<',
                                                i, re.S | re.M).group(1)
                 build.bu_id = re.search('PresellId=(.*?)$',
                                         build_url).group(1)
                 build.insert_db()
                 house_url = re.search('a href="(.*?)"', i,
                                       re.S | re.M).group(1)
                 self.get_house_info(house_url, comm.co_id, build.bu_id)
         except Exception as e:
             print('请求错误,co_index={},url={}'.format(co_index, build_url), e)
Beispiel #7
0
 def build_info(self, build_detail, co_id):
     proxy = Proxy_contact(app_name='wuhan',
                           method='get',
                           url=build_detail,
                           headers=self.headers)
     # build_res = requests.get(build_detail,headers=self.headers)
     build_res = proxy.contact()
     html = etree.HTML(build_res.decode('gb18030'))
     info_list = html.xpath("//tr[@bgcolor='#FFFFFF']")
     for info in info_list:
         try:
             bu = Building(co_index)
             bu.co_id = co_id
             bu.bu_floor = info.xpath('./td[3]/text()')[0]
             bu.bu_all_house = info.xpath('./td[4]/text()')[0]
             bu.bu_num = info.xpath('./td//span/text()')[0]
             temp_url = info.xpath('./td/a/@href')[0]
             bu.bu_id = re.search('HouseDengjh=(.*?\d+)', temp_url).group(1)
             bu.insert_db()
         except Exception as e:
             log.error('楼栋错误{}'.format(e))
             continue
         a = parse.quote(re.search('DengJh=(.*?\d+)&', temp_url).group(1),
                         encoding='gbk')
         b = parse.quote(re.search('HouseDengjh=(.*?\d+)',
                                   temp_url).group(1),
                         encoding='gbk')
         bu_url = 'http://scxx.fgj.wuhan.gov.cn/5.asp?DengJh=' + a + '&HouseDengjh=' + b
         self.house_info(bu.bu_id, bu_url, co_id)
         time.sleep(3)
Beispiel #8
0
 def get_build_info(self, co_id):
     build_url = "http://202.103.219.149:7000/ajax/LeadingMIS.CommonModel.CommonQuery.WebUI.AjaxManage.QueryDataParser,LeadingMIS.CommonModel.CommonQuery.WebUI.ashx"
     querystring = {"_method": "GetDataToDynamicInXml", "_session": "rw"}
     payload = "xmlInfo=%263Croot%2620QueryCode%263D%2622BuildingsInfo%2622%2620PageIndex%263D%26221%2622%2620PageSize%263D%262215%2622%2620SortField%263D%2622%2620ORDER%2620BY%2620Name%2622%2620QueryString%263D%2622QueryCode%263DBuildingsInfo%2626amp%263BProjectID%263D" + co_id + "%2622%2620BeginDate%263D%2622%262000%263A00%263A00%2622%2620EndDate%263D%2622%262023%263A59%263A59%2622%2620Flag%263D%2622TitleBody%2622%2620TitlesWidthInfo%263D%2622BuildNo%267C0%2624Name%267C0%2624FloorCount%267C0%2624RoomCount%267C0%2624YCJZArea%267C0%2624Structure%267C0%2624YSXKCer%267C0%2624ZJJG%267C0%2622%2620IsUseOCache%263D%26220%2622%2620IsUserID%263D%26220%2622%2620SiteId%263D%26228907bd13-1d14-4f9e-8c01-e482d9590d10%2622%2620LockedColumn%263D%26220%2622%2620IsLocked%263D%26220%2622%2620ClientWidth%263D%26221601%2622%2620ShowModeCode%263D%2622default%2622%2620Language%263D%2622chinese%2622/%263E"
     try:
         response = requests.request("POST",
                                     build_url,
                                     data=payload,
                                     params=querystring)
         html = response.text
         build_info_list = re.findall('<tr.*?>.*?</tr>', html,
                                      re.S | re.M)[1:]
         for i in build_info_list:
             build = Building(co_index)
             build.co_id = co_id
             build.bu_num = re.search(
                 '<span class="spanctfield".*?<span class="spanctfield".*?>.*?<a.*?>(.*?)<',
                 i, re.S | re.M).group(1)
             build.bu_floor = re.search(
                 '<span class="spanctfield".*?<span class="spanctfield".*?<span class="spanctfield".*?>(.*?)<',
                 i, re.S | re.M).group(1)
             build.bu_pre_sale = re.search(
                 '<span class="spanctfield".*?<span class="spanctfield".*?<span class="spanctfield".*?<span class="spanctfield".*?<span class="spanctfield".*?<span class="spanctfield".*?<span class="spanctfield".*?>(.*?)<',
                 i, re.S | re.M).group(1)
             build.bu_id = re.search('id="Tr_(.*?)"', i,
                                     re.S | re.M).group(1)
             build.insert_db()
             self.get_house_info(co_id, build.bu_id)
     except Exception as e:
         print('请求错误,url={},data={},params={}'.format(
             build_url, payload, querystring))
Beispiel #9
0
 def get_build_info(self, bu_address_list, bu_num_list, bu_floor_list,
                    bu_url_list, co_id):
     for i in range(len(bu_url_list)):
         build = Building(co_index)
         build.bu_address = bu_address_list[i]
         build.bu_num = bu_num_list[i]
         build.bu_floor = bu_floor_list[i]
         build.co_id = co_id
         # response = self.request_proxy('http://183.63.60.194:8808/public/web/' + bu_url_list[i])
         time.sleep(1)
         response = self.s.get('http://183.63.60.194:8808/public/web/' +
                               bu_url_list[i],
                               headers=self.headers)
         build.bu_id = re.search('ljzid=(.*?)$', bu_url_list[i]).group(1)
         build.insert_db()
         html = response.text
         house_html = re.search('var _table_html_.*?</script>', html,
                                re.S | re.M).group()
         house_url_list = re.findall('房屋号:<a.*?href="(.*?)"', house_html,
                                     re.S | re.M)
         try:
             self.get_house_info(house_url_list, build.bu_id)
         except Exception as e:
             print(
                 '房号错误,co_index={},url={}'.format(
                     co_index, 'http://183.63.60.194:8808/public/web/' +
                     bu_url_list[i]), e)
Beispiel #10
0
    def get_build_info(self, comm_url_list):
        for i in comm_url_list:
            try:
                sid = re.findall('\+(\d+)\+', i)[0]
                pid = re.findall('\+(\d+)\+', i)[1]
                build_url = 'http://www.jjzzfdc.com.cn/WebClient/ClientService/bldg_query.aspx?pid=' + pid + '&sid=' + sid
                # print(build_url)
                response = requests.get(build_url)
                html = response.text
                build = Building(co_index)
                build.bu_id = pid
                build.bu_num = re.search('楼栋座落.*?<td.*?>(.*?)<', html,
                                         re.S | re.M).group(1)
                build.bu_address = re.search('楼栋座落.*?<td.*?>(.*?)<', html,
                                             re.S | re.M).group(1)
                build.bu_pre_sale = re.search('预售证号.*?">(.*?)&nbsp', html,
                                              re.S | re.M).group(1)
                build.bu_pre_sale_date = re.search('时间.*?">(.*?)&nbsp', html,
                                                   re.S | re.M).group(1)
                build.bu_all_house = re.search('dM.*?">(.*?)&nbsp', html,
                                               re.S | re.M).group(1)
                # build.bu_address = re.search('售楼处地址.*?">(.*?)&nbsp', html, re.S | re.M).group(1)
                build.insert_db()
            except Exception as e:
                print('co_index={}, 楼栋错误,url={}'.format(co_index, build_url),
                      e)

            house_url = 'http://www.jjzzfdc.com.cn/WebClient/ClientService/proxp.aspx?key=WWW_LPB_001&params=' + sid
            # print(house_url)
            result = requests.get(house_url)
            html_ = result.text

            for house_info in re.findall('<Result.*?</Result>', html_,
                                         re.S | re.M):
                try:
                    house = House(co_index)
                    house.bu_id = build.bu_id
                    house.bu_num = build.bu_num
                    house.ho_name = re.search('<ONAME>(.*?)</ONAME>',
                                              house_info, re.S | re.M).group(1)
                    house.ho_num = re.search('<OSEQ>(.*?)</OSEQ>', house_info,
                                             re.S | re.M).group(1)
                    house.ho_build_size = re.search('<BAREA>(.*?)</BAREA>',
                                                    house_info,
                                                    re.S | re.M).group(1)
                    house.ho_floor = re.search('<FORC>(.*?)</FORC>',
                                               house_info,
                                               re.S | re.M).group(1)
                    house.ho_true_size = re.search('<PAREA>(.*?)</PAREA>',
                                                   house_info,
                                                   re.S | re.M).group(1)
                    house.insert_db()
                except Exception as e:
                    print('co_index={}, 房号错误'.format(co_index), e)
Beispiel #11
0
 def get_comm_info(self, url, comm):
     try:
         response = requests.get(url=url, headers=self.headers)
         html = response.text
         tree = etree.HTML(html)
         # 小区名称
         co_name = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[1]/td[2]/text()')[0].strip()
         # 小区地址
         co_address = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[2]/td[2]/text()')[0].strip()
         # 开工时间
         co_build_start_time = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[3]/td[2]/text()')[
             0].strip()
         # 竣工时间
         co_build_end_time = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[3]/td[4]/text()')[0].strip()
         # 建筑结构
         co_build_structural = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[4]/td[2]/text()')[
             0].strip()
         # 容积率
         co_volumetric = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[6]/td[4]/text()')[0].strip()
         # 绿化率
         co_green = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[6]/td[2]/text()')[0].strip()
         # 占地面的
         co_size = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[5]/td[2]/text()')[0].strip()
         co_id = re.search('home/(.*?).html', url).group(1)
         comm.co_name = co_name
         comm.co_address = co_address
         comm.co_build_start_time = co_build_start_time
         comm.co_build_end_time = co_build_end_time
         comm.co_build_structural = co_build_structural
         comm.co_volumetric = co_volumetric
         comm.co_green = co_green
         comm.co_size = co_size
         comm.co_id = co_id
         comm.insert_db()
         build_info_list = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox1"]/div/table/tr[@class="hobuild"]')
         for i in build_info_list:
             try:
                 build = Building(11)
                 # 楼栋名称
                 bu_name = i.xpath('string(td[1])')[0]
                 bu_all_house = i.xpath('td[2]/text()')[0]
                 # 楼栋id
                 bu_id = i.xpath('td[1]/strong/a/@href')[0]
                 bu_id = re.search('building_id=(.*?)$', bu_id).group(1)
                 # 建筑面积
                 bu_build_size = i.xpath('string(td[3])').replace('�O', '')
                 build.co_id = co_id
                 build.bu_id = bu_id
                 build.bu_all_house = bu_all_house
                 build.bu_name = bu_name
                 build.bu_build_size = bu_build_size
                 build.insert_db()
                 self.get_house_info(bu_id, co_id)
             except Exception as e:
                 print('楼栋错误,co_index={},url={}'.format(co_index, url), e)
     except BaseException as e:
         print('楼栋错误,co_index={},url={}'.format(co_index, url), e)
Beispiel #12
0
 def get_build_info(self, build_logo_list, preid):
     for build_logo in build_logo_list:
         try:
             build_url = 'https://www.qdfd.com.cn/qdweb/realweb/fh/FhBuildingList.jsp?preid=' + build_logo
             response = requests.get(build_url, headers=self.headers)
             html = response.text
             bu_num_list = re.findall('javascript:showHouseStatus.*?>(.*?)</a', html, re.S | re.M)
             bu_all_house_list = re.findall(
                 'javascript:showHouseStatus.*?center.*?center.*?center.*?center.*?center.*?>(.*?)<', html,
                 re.S | re.M)
             house_code_list = re.findall("javascript:showHouseStatus\((.*?)\)'>", html, re.S | re.M)
             for i in range(len(bu_num_list)):
                 try:
                     build = Building(co_index)
                     bu_code_list = re.findall('"(.*?)"', house_code_list[i])
                     build.bu_num = bu_num_list[i]
                     build.bu_all_house = bu_all_house_list[i]
                     build.co_id = preid
                     build.bu_id = bu_code_list[0]
                     build.insert_db()
                     co_id = bu_code_list[2]
                     house_id = bu_code_list[1]
                     self.get_house_info(build.bu_id, co_id, house_id)
                 except Exception as e:
                     print(e)
         except Exception as e:
             print('青岛楼栋问题,url:={}'.format(build_url), e)
Beispiel #13
0
    def build_info(self, build_url_list, co_id):
        for build_ in build_url_list:
            build_url = "http://www.njhouse.com.cn/2016/spf/" + build_

            while True:
                build_pro = Proxy_contact(app_name="nanjing",
                                          method='get',
                                          url=build_url,
                                          headers=self.headers)
                build_con = build_pro.contact()
                build_con = build_con.decode('gbk')
                html = etree.HTML(build_con)
                bu = Building(co_index)
                bu.co_id = co_id
                try:
                    bu.bu_id = re.search('buildid=(\d+)', build_).group(1)
                    bu.bu_all_house = html.xpath(
                        "//tr[@class='yll']/td/text()")[1]
                    bu.bu_num = re.search('13px;">(.*?)&nbsp&nbsp',
                                          build_con).group(1)
                    bu.insert_db()
                    house_url_list = html.xpath("//td/a[1]/@href")
                    break
                except Exception as e:
                    log.error("楼栋请求失败{}".format(e))
                    continue

            self.house_info(co_id, bu.bu_id, house_url_list)
Beispiel #14
0
 def bu_info(self, bu_list, co_id):
     for bu_ in bu_list[1:]:
         bu = Building(co_index)
         bu.co_id = co_id
         bu.bu_num = bu_.xpath("./td/a/text()")[0]
         bu.bu_pre_sale = bu_.xpath("./td[2]/text()")[0]
         bu.bu_type = bu_.xpath("./td[4]/text()")[0]
         bu_url = bu_.xpath("./td/a/@href")[0]
         bu.bu_id = re.search('buildid=(\d+)', bu_url).group(1)
         bu.insert_db()
         self.ho_info(bu_url, co_id, bu.bu_id)
Beispiel #15
0
 def get_data_obj(self, analyzer, co_index):
     if analyzer == 'comm':
         return Comm(co_index)
     elif analyzer == 'build':
         return Building(co_index)
     elif analyzer == 'house':
         return House(co_index)
Beispiel #16
0
 def build_info(self, bu_list, co_id):
     for bo in bu_list:
         ho_url = bo.xpath("./@href")[0]
         floor = bo.xpath(".//p[2]/text()")[0]
         bu = Building(co_index)
         bu.bu_pre_sale = bo.xpath(".//p[3]/text()")[0]
         bu.bu_num = re.search('zh=(.*?)', ho_url).group(1)
         bu.bu_id = re.search('n=(\d+)', ho_url).group(1)
         bu.co_id = co_id
         bu.bu_floor = re.search('总层数.*?(\d+)', floor).group(1)
         bu.insert_db()
         house_url = "http://www.ggsfcw.com/" + ho_url
         self.ho_info(house_url, co_id, bu.bu_id)
Beispiel #17
0
 def build_info(self, bu_info_list, co_id):
     for bu_info in bu_info_list:
         try:
             bu = Building(co_index)
             url = bu_info.xpath("./@onclick")[0]
             bu.bu_id = re.search('dbh=(\d+)', url).group(1)
             bu.co_id = co_id
             bu.bu_num = bu_info.xpath("./td[@class='org']/text()")[0]
             bu.bu_all_house = bu_info.xpath("./td[3]/text()")[0]
             bu.size = bu_info.xpath("./td[2]/text()")[0]
             bu.insert_db()
         except Exception as e:
             log.error('楼栋信息错误', e)
Beispiel #18
0
 def get_comm_detail(self, detail_url, area):
     try:
         comm = Comm(co_index)
         comm_detail_url = 'http://www.yfci.gov.cn:8080/HousePresell/' + detail_url
         response = requests.get(comm_detail_url, headers=self.headers)
         html = response.text
         comm.co_develops = re.search('id="kfsmc".*?<a.*?>(.*?)<', html,
                                      re.S | re.M).group(1)
         comm.co_name = re.search('id="PresellName".*?<a.*?>(.*?)<', html,
                                  re.S | re.M).group(1)
         comm.co_address = re.search('id="HouseRepose".*?>(.*?)<', html,
                                     re.S | re.M).group(1)
         comm.co_build_size = re.search('id="PresellArea".*?>(.*?)<', html,
                                        re.S | re.M).group(1)
         comm.co_all_house = re.search('id="djrqtd".*?>(.*?)<', html,
                                       re.S | re.M).group(1)
         comm.co_land_use = re.search('id="landinfo".*?>(.*?)<', html,
                                      re.S | re.M).group(1)
         comm.co_type = re.search('id="zczjtd".*?>(.*?)<', html,
                                  re.S | re.M).group(1)
         comm.co_pre_sale = re.search('id="bookid".*?<a.*?>(.*?)<', html,
                                      re.S | re.M).group(1)
         comm.co_pre_sale_date = re.search('id="FZDatebegin".*?>(.*?)<',
                                           html, re.S | re.M).group(1)
         comm.co_open_time = re.search('id="kpdate".*?>(.*?)<', html,
                                       re.S | re.M).group(1)
         comm.co_id = re.search('FD=(.*?)&', detail_url,
                                re.S | re.M).group(1)
         comm.area = area
         comm.insert_db()
         build_html = re.search('id="donglist".*?</table>', html,
                                re.S | re.M).group()
         build_info_list = re.findall('<tr.*?</tr>', build_html,
                                      re.S | re.M)
         for i in build_info_list:
             build = Building(co_index)
             build.co_id = comm.co_id
             build.bu_address = re.search('<td.*?<td.*?>(.*?)<', i,
                                          re.S | re.M).group(1)
             build.bu_num = re.search('<td.*?<td.*?<td.*?>(.*?)<', i,
                                      re.S | re.M).group(1)
             build.bu_floor = re.search('<td.*?<td.*?<td.*?<td.*?>(.*?)<',
                                        i, re.S | re.M).group(1)
             house_url = re.search('href="(.*?)"', i, re.S | re.M).group(1)
             build.bu_id = re.search("LID=(.*?)$", house_url,
                                     re.S | re.M).group(1)
             build.insert_db()
             self.get_house_info(house_url, comm.co_id, build.bu_id)
     except Exception as e:
         print('小区错误,co_index={},url={}'.format(co_index, comm_detail_url),
               e)
Beispiel #19
0
 def get_comm_info(self, comm_id_list):
     for i in comm_id_list:
         try:
             comm = Comm(co_index)
             comm_url = 'http://web.xxfdc.gov.cn/onlineQuery/projectInformation.do?xmId=' + i
             response = requests.get(comm_url, headers=self.headers)
             html = response.text
             comm.co_name = re.search('项目名称:.*?<td.*?>(.*?)<', html,
                                      re.S | re.M).group(1)
             comm.co_address = re.search('项目地址:.*?<td.*?>(.*?)<', html,
                                         re.S | re.M).group(1)
             comm.co_develops = re.search('开发商:.*?<td.*?>(.*?)<', html,
                                          re.S | re.M).group(1)
             comm.co_all_house = re.search('已售总套数:.*?<td.*?>(.*?)<', html,
                                           re.S | re.M).group(1)
             comm.co_build_size = re.search('已售总面积:.*?<td.*?>(.*?)<', html,
                                            re.S | re.M).group(1)
             comm.area = re.search('行政区别:.*?<td.*?>(.*?)<', html,
                                   re.S | re.M).group(1)
             comm.co_volumetric = re.search('容积率:.*?<td.*?>(.*?)<', html,
                                            re.S | re.M).group(1)
             comm.co_id = i
             comm.insert_db()
             bu_html = re.search(
                 '<table class="table table-bordered itemInfoDetail.*?</table>',
                 html, re.S | re.M).group()
             build_info_list = re.findall('<tr>.*?</tr>', bu_html,
                                          re.S | re.M)[1:]
             for i in build_info_list:
                 try:
                     build = Building(co_index)
                     build.bu_num = re.search('<td>(.*?)<', i,
                                              re.S | re.M).group(1)
                     build.bu_all_house = re.search(
                         '<td>.*?<td>.*?<td>(.*?)<', i,
                         re.S | re.M).group(1)
                     build.bu_id = re.search('buildId=(.*?)&', i,
                                             re.S | re.M).group(1)
                     build.co_id = comm.co_id
                     build.insert_db()
                     house_url = re.search('<a href="(.*?)"', bu_html,
                                           re.S | re.M).group(1)
                     response = requests.get(house_url,
                                             headers=self.headers)
                     html = response.text
                     house_url_list = re.findall(
                         '<td width="110">.*?<a.*?href="(.*?)"', html,
                         re.S | re.M)
                     self.get_house_info(house_url_list, build.bu_id,
                                         comm.co_id)
                 except Exception as e:
                     print(
                         '楼栋错误,co_index={},url={}'.format(
                             co_index, house_url), e)
         except Exception as e:
             print('小区错误,co_index={},url={}'.format(co_index, comm_url), e)
Beispiel #20
0
    def co_parse(self,url_list):
        for url in url_list:
            try:
                co_url = url.xpath("./@href")[0]
                new_url = "http://tmsf.qzfdcgl.com" + co_url
                co_res = requests.get(new_url,headers=self.headers)
                con = co_res.text
                co = Comm(co_index)
                co.co_id = re.search('property_(.*?)_info',co_url).group(1)
                co.co_name = re.search('楼盘名称:</span>(.*)',con).group(1)
                co.co_develops = re.search('项目公司:</span>(.*)',con).group(1)
                co.co_address = re.search('物业地址:</span>(.*?)</p',con,re.S|re.M).group(1)
                co.area = re.search('所属城区:</span>(.*)',con).group(1)
                co.insert_db()
                sid = re.search('property_(\d+)_',co_url).group(1)
                propertyid = re.search('(\d+)_info',co_url).group(1)
                bu_url = new_url.replace('info','price')
                res = requests.get(bu_url,headers=self.headers)
                bu_html = etree.HTML(res.text)
                bu_idlist = bu_html.xpath("//dd[@id='building_dd']/a")
            except:
                continue
            for bu_ in bu_idlist[1:]:
                id = bu_.xpath("./@id")[0]
                bu_id = re.search('.*?(\d+)',id).group(1)
                bu = Building(co_index)
                bu.bu_id = bu_id
                bu.co_id = co.co_id
                bu.bu_num = bu_.xpath("./text()")[0]

                bu.insert_db()
                self.house_parse(bu_id,co.co_id,sid,propertyid)
Beispiel #21
0
 def get_comm_info(self, comm_url_list):
     for i in comm_url_list:
         comm_url = "http://www.kmhouse.org" + i
         try:
             co_id = re.search("PreId=(.*?)&", i).group(1)
             s = re.search('prename=(.*?)$', comm_url, re.S | re.M).group(1)
             s_decode_str = parse.quote(s, encoding='gbk')
             comm_url = comm_url.replace(s, s_decode_str)
             response = requests.get(comm_url, headers=self.headers)
             html = response.content.decode('gbk')
             comm_detail_url = re.findall('linkone" href="(.*?)"', html,
                                          re.S | re.M)[0]
             self.get_comm_detail(comm_detail_url)
             build_html = re.search('请选择幢号.*?</select>', html,
                                    re.S | re.M).group()
             bu_info_list = re.findall("<option.*?</option>", build_html,
                                       re.S | re.M)
             for info in bu_info_list:
                 build = Building(co_index)
                 build.bu_id = re.search("value='(.*?)'", info,
                                         re.S | re.M).group(1)
                 build.bu_num = re.search("<option.*?>(.*?)<", info,
                                          re.S | re.M).group(1)
                 build.co_id = co_id
                 build.insert_db()
                 self.get_build_info(build.bu_id, co_id)
         except Exception as e:
             print('小区错误,co_index={},url={}'.format(co_index, comm_url), e)
Beispiel #22
0
 def bu_parse(self, detail_url, co_id):
     pre_url = detail_url.replace('lp', 'presell')
     pre_res = requests.get(pre_url, headers=self.headers)
     pre_html = etree.HTML(pre_res.text)
     bu_pre_list = pre_html.xpath("//dt/strong/a")
     for bu_pre in bu_pre_list:
         bu_pre_url = bu_pre.xpath("./@href")[0]
         bu_pre_sale = bu_pre.xpath("./text()")[0]
         bu_url = 'http://www.zstmsf.com' + bu_pre_url
         while True:
             try:
                 proxy = self.proxies[random.randint(0, 9)]
                 bu_res = requests.get(bu_url,
                                       headers=self.headers,
                                       proxies=proxy,
                                       timeout=10)
                 break
             except:
                 continue
         bu_html = etree.HTML(bu_res.text)
         bu_list = bu_html.xpath("//tr//strong/a/@href")
         for bo_url in bu_list:
             ho_url = "http://www.zstmsf.com" + bo_url
             while True:
                 try:
                     proxy = self.proxies[random.randint(0, 9)]
                     ho_res = requests.get(ho_url,
                                           headers=self.headers,
                                           proxies=proxy,
                                           timeout=10)
                     break
                 except:
                     continue
             build = Building(co_index)
             build.co_id = co_id
             build.bu_id = re.search('zid=.*?(\d+)', ho_url).group(1)
             build.bu_num = re.search('幢名称:<strong>(.*?)<',
                                      ho_res.text).group(1)
             build.bu_all_house = re.search("幢总套数.*?'>(.*?)</",
                                            ho_res.text).group(1)
             build.bu_all_size = re.findall("面积.*?'>(.*?)</",
                                            ho_res.text)[0]
             build.bu_pre_sale = bu_pre_sale
             build.insert_db()
             self.ho_parse(co_id, build.bu_id, ho_res)
Beispiel #23
0
 def get_build_info(self, build_info_list, co_id):
     for i in build_info_list:
         try:
             build = Building(co_index)
             build.bu_num = re.search('<td>(.*?)</td>', i, re.S | re.M).group(1)
             build.bu_all_house = re.search('<td>.*?<td>(.*?)</td>', i, re.S | re.M).group(1)
             build.bu_all_size = re.search('<td>.*?<td>.*?<td>(.*?)</td>', i, re.S | re.M).group(1)
             build.bu_id = re.search('\?id=(.*?)"', i, re.S | re.M).group(1)
             build.co_id = co_id
             build.insert_db()
             house_url = re.search('href="(.*?)"', i, re.S | re.M).group(1)
             self.get_house_info(house_url, co_id, build.bu_id)
         except Exception as e:
             print('楼栋错误,co_index={},str={}'.format(co_index, i), e)
Beispiel #24
0
    def get_build_info(self, build_url_list, bu_pre_sale_list, co_name, co_id):
        for i in range(len(build_url_list)):
            try:
                build = Building(co_index)
                build.co_id = co_id

                build.co_name = co_name
                build.bu_pre_sale = bu_pre_sale_list[i]
                build.bu_id = re.search('lh=(\d+)', build_url_list[i]).group(1)
                build_url = 'http://221.2.144.162:8090/' + build_url_list[i]
                response = requests.get(build_url, headers=self.headers)
                html = response.content.decode('gbk')
                build.bu_num = re.findall('<font color=white.*?><b>(.*?)<',
                                          html, re.S | re.M)[0]
                build.bu_address = re.findall('坐落位置:</b>(.*?)<', html,
                                              re.S | re.M)[0]
                build.insert_db()
                ho_url_list = re.findall('background-.*?href=(.*?) ', html,
                                         re.S | re.M)
                ho_name_list = re.findall('background-color.*?<a.*?>(.*?)<',
                                          html, re.S | re.M)
                for i in range(len(ho_url_list)):
                    try:
                        house = House(co_index)
                        house_url = 'http://221.2.144.162:8090/' + ho_url_list[
                            i]
                        result = requests.get(
                            house_url,
                            headers=self.headers).content.decode('gbk')
                        house.bu_id = build.bu_id
                        house.co_id = co_id
                        house.ho_type = re.findall(
                            '用&nbsp;&nbsp;&nbsp;途:.*?<td.*?>(.*?)<', result,
                            re.S | re.M)[0]
                        house.ho_build_size = re.findall(
                            '建筑面积:.*?<td>(.*?)<', result, re.S | re.M)[0]
                        house.bu_num = build.bu_num
                        house.co_name = co_name
                        house.ho_name = ho_name_list[i]
                        house.insert_db()
                    except Exception as e:
                        print("co_index={},房屋信息错误".format(co_index), e)
            except Exception as e:
                print("co_index={},楼栋信息错误".format(co_index), e)
Beispiel #25
0
 def get_comm_info(self, comm_info_list):
     for i in comm_info_list:
         build = Building(co_index)
         house_url = re.search('href="(.*?)"', i, re.S | re.M).group(1)
         build.bu_num = re.search('<a.*?>(.*?)<', i, re.S | re.M).group(1)
         build.bu_address = re.search('<td.*?<td.*?>(.*?)<', i,
                                      re.S | re.M).group(1)
         build.bu_pre_sale = re.search('<td.*?<td.*?<td.*?>(.*?)<', i,
                                       re.S | re.M).group(1)
         build.bu_id = re.search('slbh=(.*?)&', i, re.S | re.M).group(1)
         build.insert_db()
         self.get_house_info(house_url, build.bu_id)
Beispiel #26
0
 def build_parse(self, co_id):  # 楼栋信息解析
     bu = Building(co_index)
     build_info_url = "http://ys.tyfdc.gov.cn/Firsthand/tyfc/publish/ProNBList.do"
     formdata = {"pid": co_id, "pageNo": "1", "pageSize": "50"}
     res = requests.post(build_info_url,
                         data=formdata,
                         headers=self.headers)
     con = res.text
     info = re.findall('<tr objid.*?</tr>', con, re.S | re.M)
     for i in info:
         bu.co_id = co_id
         bu.bu_id = re.search('objid="(\d+)"', i).group(1)
         bu.bu_num = re.findall('<span>(.*?)<', i)[1]
         bu.bu_floor = re.search('<td>(\d+)\(', i).group(1)
         bu.bu_address = re.findall('<td>(.*?)</td>', i)[-1]
         bu.insert_db()
         self.house_parse(bu.bu_id, co_id)
Beispiel #27
0
    def get_comm_info(self, comm_url_list):
        for i in comm_url_list:
            comm = Comm(co_index)
            comm_url = 'http://old.newhouse.cnnbfdc.com/' + i
            try:
                response = requests.get(comm_url, headers=self.headers)
            except Exception as e:
                print("{}城市无法访问小区{}".format(city, comm_url), e)
                continue

            html = response.text
            con = etree.HTML(html)
            comm.co_id = re.search('id=(\d+)', i).group(1)
            comm.co_name = re.findall('项目名称:.*?<span.*?>(.*?)<', html,
                                      re.S | re.M)[0]
            comm.co_address = re.findall('项目地址:.*?<td.*?>(.*?)<', html,
                                         re.S | re.M)[0]
            comm.co_develops = re.findall('开发公司:.*?<td.*?>(.*?)<', html,
                                          re.S | re.M)[0]
            comm.co_pre_sale = re.findall('售证名称:.*?<td.*?>(.*?)<', html,
                                          re.S | re.M)[0]
            comm.co_build_size = re.findall('纳入网上可售面积:.*?<img.*?>(.*?)<', html,
                                            re.S | re.M)[0]
            comm.co_all_house = re.findall('纳入网上可售套数:.*?<img.*?>(.*?)<', html,
                                           re.S | re.M)[0]
            comm.area = re.findall('所在区县:.*?<td.*?>(.*?)<', html,
                                   re.S | re.M)[0]
            comm.insert_db()
            bu_all_house_list = re.findall(
                'window.open.*?center.*?center.*?>(.*?)<', html, re.S | re.M)
            try:
                bu_url_list = re.findall("window\.open\('(.*?)'", html,
                                         re.S | re.M)
            except Exception as e:
                print("{}城市{}小区无楼栋".format(city, comm.co_name), e)
                continue
            for i in range(len(bu_url_list)):
                build = Building(co_index)
                bu_url = bu_url_list[i]
                build.bu_all_house = bu_all_house_list[i]
                build.co_name = comm.co_name
                build.bu_num = con.xpath("//a[@href='#']/@title")[i]
                build.bu_id = re.search('key=(\d+)&', bu_url).group(1)
                build.co_id = comm.co_id
                build.insert_db()
                self.get_house_info(bu_url, build.bu_id)
Beispiel #28
0
 def get_build_info(self, build_info_list, bu_num_list, co_id, comm_url):
     for i in range(len(bu_num_list)):
         try:
             build = Building(co_index)
             build.co_id = co_id
             build.info = build_info_list[i]
             build.bu_num = bu_num_list[i]
             build.insert_db()
         except Exception as e:
             print('楼栋错误,co_index={},url={}'.format(co_index, comm_url), e)
Beispiel #29
0
 def get_build_info(self, build_url_list, co_id):
     bu_code_list = build_url_list[0].split(';;')
     for i in bu_code_list:
         build = Building(co_index)
         build.co_id = co_id
         code = i.split(',,')
         build.bu_id = code[0]
         build.bu_num = code[1]
         build.insert_db()
         self.get_house_info(build.bu_id)
Beispiel #30
0
    def build_info(self, build_url_list):
        bu = Building(co_index)

        for build_url in build_url_list:
            url = "http://ris.szpl.gov.cn/bol/" + build_url
            res = requests.get(url, headers=self.headers)
            con = etree.HTML(res.text)
            branch_list = con.xpath("//div[@id='divShowBranch/a/@href")
            for branch in branch_list:
                branch_url = "http://ris.szpl.gov.cn/bol/" + branch
                response = requests.get(branch_url, headers=self.headers)

                content = etree.HTML(response.text)
                bu.bu_num = content.xpath(
                    "//div[@id='curAddress']/a/text()")[2]
                bu.co_name = content.xpath(
                    "//div[@id='curAddress']/a/text()")[1]
                co_info = content.xpath("//form/@action")[0]
                bu.bu_id = bu_id = re.search('\?id=(\d+)&', co_info).group(1)
                bu.co_id = co_id = re.search('presellid=(\d+)&',
                                             co_info).group(1)

                bu.insert_db()
                house_list = content.xpath(
                    "//div[@id='updatepanel1']//tr[@class='a1']//a/@href"
                )[2:-1]

                self.house_info(house_list, bu_id, co_id)