def fetch_list_pages(db, city_index): logging.info('抓取区域列表') areas_obj = db.query(Area).filter(Area.city_index == city_index, Area.insert == 0).all() for area in areas_obj: page = 1 while True: logging.info(f'正在抓取第{page}页') api_url = f'http://m.58.com/xiaoquweb/getXiaoquList/?city={area.area_index}&page={page}' res = requests.get(api_url, headers=config.config.headers, timeout=5) res.raise_for_status() res_json_obj = json.loads(res.content) if res_json_obj['code'] == "0": page_dto = res_json_obj['data']['pageDTO'] info_list = res_json_obj['data']['infoList'] if page_dto['currentPageNo'] > page_dto['totalPage']: break for community in info_list: add_community = Community(city_index, area.area_index, community) db.add(add_community) logging.info('新增1条小区信息') page += 1 area.insert = 1 db.commit() logging.info('抓取区域列表数据完毕') db.close()
def add_community(self): ''' Create list_of_community built of Community objects. Except Vallue Error when data isn't integer. ''' first_index_community = 0 for row in Territory.list_of_territory: try: index_community = int(row[2]) if index_community != first_index_community and self.number == int( row[1]): community = Community(row[4], row[5], index_community) self.list_of_community.append(community) first_index_community = index_community except ValueError: continue
def load_file(): if not os.path.isfile('static/malopolska.csv'): raise FileNotFoundError else: with open('static/malopolska.csv', 'r') as csvfile: for row in csvfile: row = row.strip() row = row.split('\t') if row[0].isdigit(): data = (row[4], row[5]) if not row[1]: voivodeship = Voivodeship(row[0], *data) elif not row[2]: county = County(row[1], *data) voivodeship.add_county(county) elif row[3]: community = Community(row[2], row[3], *data) county.add_community(community)