def getIpstate():
ipstate = RedisUtils.get("ipstate")
if (ipstate):
return "%s" % ipstate
else:
RedisUtils.set("ipstate", 1)
return "1"
def checkProxy(self):
RedisUtils.flush()
cookies = urllib2.HTTPCookieProcessor()
for proxy in self.proxyList:
proxyHandler = urllib2.ProxyHandler(
{"http": r'http://%s:%s' % (proxy[0], proxy[1])})
opener = urllib2.build_opener(cookies, proxyHandler)
opener.addheaders = [(
'User-agent',
'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:22.0) Gecko/20100101 Firefox/22.0'
)]
t1 = time.time()
try:
req = opener.open(self.testUrl, timeout=self.timeout)
result = req.read()
timeused = time.time() - t1
pos = result.find(self.testStr)
if pos > 1 and int(timeused) < 20:
checkedProxyList.append((proxy[0], proxy[1], timeused))
RedisUtils.set("ip1-%s" % proxy[0],
proxy[0] + ":" + proxy[1])
logger.info("ip %s" % proxy[0])
else:
continue
except Exception, e:
continue
def delIp(ip):
if (ip and len(ip) > 0):
value = ip.get("http")
value = value.replace("http://", "")
nPos = value.index(":")
if (nPos > 0):
ip = value[0:nPos]
ipstate = int(getIpstate()) + 1
RedisUtils.remove("ip%s-%s" % (ipstate, ip))
logger.info("删除代理ip %s-%s" % (ipstate, ip))
def removeIp(ip):
if (ip and len(ip) > 0):
value = ip.get("http")
value = value.replace("http://", "")
nPos = value.index(":")
if (nPos > 0):
ip = value[0:nPos]
RedisUtils.remove("ip%s-%s" % (getIpstate(), ip))
ipstate = int(getIpstate()) + 1
RedisUtils.set("ip%s-%s" % (ipstate, ip), value)
def process_proxy():
currentKey = getIpstate();
key = "ip%s-" % currentKey
keys = RedisUtils.getKeys(key)
keyNum = len(keys)
if (keyNum < 1):
logger.info("代理池异常数量<1 sleepping 120s")
RedisUtils.set("ipstate", int(currentKey) + 1)
time.sleep(1200)
key = "ip%s-" % getIpstate()
keys = RedisUtils.getKeys(key)
keyNum = len(keys)
randNum = random.randint(0, keyNum - 1)
current = keys[randNum]
value = RedisUtils.get(current)
proxies = {}
proxies["http"] = r'http://%s' % value
removeIp(proxies)
return proxies
from flask import Flask
from flask import request
from Utils import RedisUtils
from DataSpider.DataSpider.begin import amap_spider
import requests
import json
app = Flask(__name__)
redis_ = RedisUtils.get_conn()
@app.route('/cityAndWeather', methods=['GET'])
def get_location():
key = redis_.get("amap_key")
location_last_url = "https://restapi.amap.com/v3/ip?ip=%s&output=json&key=%s" % (
request.args.get('lastIp'), key)
location_now_url = "https://restapi.amap.com/v3/ip?ip=%s&output=json&key=%s" % (
request.args.get('ip'), key)
location_last = requests.get(location_last_url).json()
location_now = requests.get(location_now_url).json()
last_city = location_last['city']
city_code = location_now['adcode']
weather_url_live = "https://restapi.amap.com/v3/weather/weatherInfo?city=%s&extensions=base&key=%s" % (
city_code, key)
weather_url_forecast = "https://restapi.amap.com/v3/weather/weatherInfo?city=%s&extensions=all&key=%s" % (
city_code, key)
weather_live = requests.get(weather_url_live).json()
weather_forecast = requests.get(weather_url_forecast).json()
result = json.dumps({
"city": last_city,