def download_page(self, base_url, geo, proxy, city_code, sec, page_table): downloader = PageDownload(proxy=proxy, hd=mobike_headers) post_dic = { 'longitude': str(geo[0]), 'latitude': str(geo[1]), 'citycode': str(city_code), 'errMsg': 'getMapCenterLocation:ok' } page = downloader.download_with_post(url=base_url, post_data=post_dic) if is_json(page): json_page = json.loads(page) if json_page.has_key("object"): mobike_object = json_page["object"] items = [] for mobike in mobike_object: bike_id = mobike["distId"] bike_type = mobike["biketype"] b_type = mobike["type"] lng = mobike["distX"] lat = mobike["distY"] dis_source = str(geo[0]) + "," + str(geo[1]) item = (bike_id, bike_type, b_type, lat, lng, dis_source, sec) items.append(item) db = MysqlHandle() sql = "insert into " + page_table + " values(%s,%s,%s,%s,%s,%s,%s,now())" db.insert(sql=sql, value_list=items) return True else: return False
def filter_avaliable_ips(): db = MysqlHandle() is_success = db.delete('DELETE FROM TEMP_IPS_MANAGE') if not is_success: db.close() return db.close() sql = 'SELECT PROXY FROM AI_PROXY_IPS' db = MysqlHandle() # 查询出所有代理ip IP_LIST = db.query(sql) db.close() for ip in IP_LIST: PROXY = {'http': 'http://' + ip[0]} # 代理 print 'filtering ip:' + ip[0] downloader = PageDownload(hd=mobike_headers, proxy=PROXY, timeout=3) try: post_data = { 'longitude': '121.1883', 'latitude': '31.05147', 'citycode': '021', 'errMsg': 'getMapCenterLocation:ok' } page = downloader.download_with_post(url=TEST_URL, post_data=post_data) if page is not None: AVALIABLE_IPS.append(ip) print ip[0] + " is ok!" else: pass except Exception, e: print str(e) pass
def filter_avaliable_ips(): db = MysqlHandle() is_success = db.delete('DELETE FROM TEMP_IPS_MANAGE') if not is_success: db.close() return db.close() sql = 'SELECT PROXY FROM AI_PROXY_IPS' db = MysqlHandle() # 查询出所有代理ip IP_LIST = db.query(sql) db.close() for ip in IP_LIST: PROXY = {'http': 'http://'+ip[0]} # 代理 print 'filtering ip:'+ip[0] downloader = PageDownload(hd=ofo_headers,proxy=PROXY) try: post_data = MultipartEncoder( { "lat": "30.515133", "lng": "114.346161", "token": "7eb9b200-3d7f-11e8-b714-e9d19c19f7b0", "source": "-5", "source-version": "10005" }, boundary='----ofo-boundary-MC40MjcxMzUw' ) page = downloader.download_with_post(url=TEST_URL,post_data=post_data) if page is not None: AVALIABLE_IPS.append(ip) print ip[0]+" is ok!" else: pass except Exception, e: print str(e) pass