def download_page(self, base_url, geo, proxy, city_code, sec, page_table):
downloader = PageDownload(proxy=proxy, hd=mobike_headers)
post_dic = {
'longitude': str(geo[0]),
'latitude': str(geo[1]),
'citycode': str(city_code),
'errMsg': 'getMapCenterLocation:ok'
}
page = downloader.download_with_post(url=base_url, post_data=post_dic)
if is_json(page):
json_page = json.loads(page)
if json_page.has_key("object"):
mobike_object = json_page["object"]
items = []
for mobike in mobike_object:
bike_id = mobike["distId"]
bike_type = mobike["biketype"]
b_type = mobike["type"]
lng = mobike["distX"]
lat = mobike["distY"]
dis_source = str(geo[0]) + "," + str(geo[1])
item = (bike_id, bike_type, b_type, lat, lng, dis_source,
sec)
items.append(item)
db = MysqlHandle()
sql = "insert into " + page_table + " values(%s,%s,%s,%s,%s,%s,%s,now())"
db.insert(sql=sql, value_list=items)
return True
else:
return False
def filter_avaliable_ips():
db = MysqlHandle()
is_success = db.delete('DELETE FROM TEMP_IPS_MANAGE')
if not is_success:
db.close()
return
db.close()
sql = 'SELECT PROXY FROM AI_PROXY_IPS'
db = MysqlHandle()
# 查询出所有代理ip
IP_LIST = db.query(sql)
db.close()
for ip in IP_LIST:
PROXY = {'http': 'http://' + ip[0]} # 代理
print 'filtering ip:' + ip[0]
downloader = PageDownload(hd=mobike_headers, proxy=PROXY, timeout=3)
try:
post_data = {
'longitude': '121.1883',
'latitude': '31.05147',
'citycode': '021',
'errMsg': 'getMapCenterLocation:ok'
}
page = downloader.download_with_post(url=TEST_URL,
post_data=post_data)
if page is not None:
AVALIABLE_IPS.append(ip)
print ip[0] + " is ok!"
else:
pass
except Exception, e:
print str(e)
pass
def filter_avaliable_ips():
db = MysqlHandle()
is_success = db.delete('DELETE FROM TEMP_IPS_MANAGE')
if not is_success:
db.close()
return
db.close()
sql = 'SELECT PROXY FROM AI_PROXY_IPS'
db = MysqlHandle()
# 查询出所有代理ip
IP_LIST = db.query(sql)
db.close()
for ip in IP_LIST:
PROXY = {'http': 'http://'+ip[0]} # 代理
print 'filtering ip:'+ip[0]
downloader = PageDownload(hd=ofo_headers,proxy=PROXY)
try:
post_data = MultipartEncoder(
{
"lat": "30.515133",
"lng": "114.346161",
"token": "7eb9b200-3d7f-11e8-b714-e9d19c19f7b0",
"source": "-5",
"source-version": "10005"
},
boundary='----ofo-boundary-MC40MjcxMzUw'
)
page = downloader.download_with_post(url=TEST_URL,post_data=post_data)
if page is not None:
AVALIABLE_IPS.append(ip)
print ip[0]+" is ok!"
else:
pass
except Exception, e:
print str(e)
pass