def new_view(request, *args, **kwargs):
if 'token' in request.session and 'expire' in request.session:
client = APIClient(app_key=APP_KEY, app_secret=APP_SECRET, redirect_uri=CALLBACK_URL)
client.set_access_token(request.session['token'], request.session['expire'])
if client.is_expires():
return render(request, 'loginForm.html')
return view(request, client, *args, **kwargs)
return render(request, 'loginForm.html')
class weiboNoAuthClientTest(unittest.TestCase):
def setUp(self):
self.client = APIClient()
self.app_key = "1865902151"
def test_isExpired(self):
self.assertFalse(self.client.is_expires())
def test_format(self):
self.assertEqual(self.client.format, "json")
def test_apitype(self):
self.assertEqual(self.client.api_url, "https://api.weibo.com/2/")
def index(request):
client = APIClient(app_key=APP_KEY, app_secret=APP_SECRET, redirect_uri=CALLBACK_URL)
requestResults = dict()
if "code" in request.GET:
print("in request code, set_cookie")
code = request.GET['code']
r = client.request_access_token(code)
access_token = r.access_token
expires_in = r.expires_in
client.set_access_token(access_token, expires_in)
elif "access_token" in request.COOKIES:
print("has access_token in cookies")
access_token = request.COOKIES["access_token"]
expires_in = request.COOKIES["expires_in"]
client.set_access_token(access_token, expires_in)
if not client.is_expires():
print("client is not expires, get information")
accountUid = client.get.account__get_uid()
usersShow = client.get.users__show(uid=accountUid["uid"])
requestResults["usersShow"] = usersShow
else:
print("client is expire, authorize again")
authorizeUrl = client.get_authorize_url()
requestResults["authorizeUrl"] = authorizeUrl
response = render_to_response("amour/index.html", requestResults)
if client.is_expires():
print("client is expires, clear cookie")
response.delete_cookie("access_token")
response.delete_cookie("expires_in")
else:
print("client is not expires, set cooke")
response.set_cookie("access_token", access_token)
response.set_cookie('expires_in', expires_in)
return response
class weiboAPI(object):
'''
该类是线程安全的，特别是需要使用weibo链接网络的方法一定要加锁，还有就是判断访问频次的方法也要加锁
如果要用户授权，要保证授权未过期！！
'''
def __init__(self, access_token=None, expires_in=None, u_id=None):
self.client = APIClient(app_key=APP_KEY,
app_secret=APP_SECRET,
redirect_uri=CALLBACK_URL,
)
self.u_id = u_id
self.REMIND_WEIBO_ID = 3504267275499498
self.lock = threading.Lock() # #多线程同步
self.acc_count = {}
self.acc_limit = {}
self._initAccessCount()
self._initAccessLimit()
if access_token is not None and expires_in is not None:
self.client.set_access_token(access_token, expires_in)
self.refreshAccessToken()
logger.info('current user:'******'\naccess_token:' + str(access_token) + \
'\nexpires_in' + str(expires_in))
else:
logger.info('just weibo client, no user authorized')
def _initAccessCount(self):
'''
该方法没有加锁，要求调用的方法必须是线程安全的
'''
self.acc_count['hour'] = time.localtime().tm_hour
self.acc_count['ip_all'] = 0 # 单小时限制:10000
self.acc_count['user_all'] = 0 # 单小时限制:1000
self.acc_count['user_status'] = 0 # 单小时限制:30
self.acc_count['user_comment'] = 0 # 单小时限制:60
self.acc_count['user_follow'] = 0 # 单小时限制:60;这个还有每天的限制，没有考虑
logger.info('_initAccessCount:\t' + str(self.acc_count))
def _initAccessLimit(self):
'''
该方法没有加锁，要求调用的方法必须是线程安全的
'''
self.acc_limit['time_unit'] = 'hour'
self.acc_limit['ip_all'] = 10000 - 100 # #减去这些值是为了考虑到有漏掉的访问
self.acc_limit['user_all'] = 1000 - 20
self.acc_limit['user_status'] = 30 - 5
self.acc_limit['user_comment'] = 60 - 10
self.acc_limit['user_follow'] = 60 - 10 # 这个还有每天的限制，没有考虑
logger.info('_initAccessLimit:\t' + str(self.acc_limit))
def _checkAccessLimit(self, type='ip_all'):
'''
线程安全
'''
with self.lock:
logger.info('_checkAccessLimit, type:' + str(type))
logger.info('old acc_count:\t' + str(self.acc_count))
# #MARK: 两小时刷新一次记录会不会过分了
if self.acc_count['hour'] < time.localtime().tm_hour or \
self.acc_count['hour'] > time.localtime().tm_hour + 1:
self._initAccessCount()
else:
self.acc_count[type] += 1
if type != 'ip_all':
self.acc_count['ip_all'] += 1
self.acc_count['user_all'] += 1
logger.info('new acc_count:\t' + str(self.acc_count))
if self.acc_count[type] > self.acc_limit[type] or \
self.acc_count['ip_all'] > self.acc_limit['ip_all'] or \
self.acc_count['user_all'] > self.acc_limit['user_all']:
sleeptime = (60 - time.localtime().tm_min + 65) * 60 # #MARK目前多睡１小时。。。
logger.info('access limit reached, sleep for ' + str(sleeptime / 60) + ' minutes')
time.sleep(sleeptime)
def refreshAccessToken(self):
'''
这个方法实现自动授权？
'''
logger.info('refreshAccessToken')
if self.client.is_expires():
logger.error('refreshAccessToken: access_token is expired')
self._setAccessTokenManually()
def _setAccessTokenManually(self):
url = self.client.get_authorize_url()
print 'weibo.refreshAccessToken 访问url获取认证code\n', url
# TODO:能否impl自动认证
code = raw_input()
r = self.client.request_access_token(code)
access_token = r.access_token # 新浪返回的token，类似abc123xyz456
expires_in = r.expires_in # token过期的UNIX时间：http://zh.wikipedia.org/wiki/UNIX%E6%97%B6%E9%97%B4
self.client.set_access_token(access_token, expires_in)
def getAuthorizeUrl(self):
'''该方法没有访问服务器，只是静态的字符串拼接，不需要加锁'''
return self.client.get_authorize_url()
def getWeeklyHotTopics(self):
'''得到一周的热门话题'''
self._checkAccessLimit()
with self.lock:
return self.client.get.trends__weekly()['trends']
def getDailyHotTopics(self):
'''得到一周的热门话题'''
self._checkAccessLimit()
with self.lock:
return self.client.get.trends__daily()['trends']
def getUID(self):
'''得到当前授权用户的ID
TODO: 添加判断授权的机制
'''
self._checkAccessLimit()
with self.lock:
return self.client.get.account__get_uid()['uid']
def getUserInfo(self, uid=None):
''' 获取用户信息
如果uid非空，请求uid的用户信息
如果uid是空，则请求当前授权用户的用户信息
'''
self._checkAccessLimit()
with self.lock:
if uid is not None:
_uid = uid
elif self.u_id is None:
_uid = self.getUID()
else:
_uid = self.u_id
return self.client.get.users__show(uid=_uid)
def getMentions(self, count=50, page=1, since_id=0, trim_user=0):
''' 得到当前授权用户的mention信息 '''
self._checkAccessLimit()
with self.lock:
return self.client.get.statuses__mentions(count=count, \
page=page, \
since_id=since_id, \
trim_user=trim_user)
def getRelevantWeibo(self, topicTitle, count=10, page=1):
'''
得到和topicTitle相关的最新微薄
count [0,50]
'''
self._checkAccessLimit()
with self.lock:
return self.client.get.search__topics(q=topicTitle, \
count=count, \
page=page)
def postComment(self, weibo_id, content):
''' 发布评论 '''
self._checkAccessLimit('user_comment')
with self.lock:
if len(content) > 139:
content = content[:139]
return self.client.post.comments__create(comment=content, id=weibo_id)
def repostStatus(self, weibo_id, content, is_comment=3):
''' 转发微博
一些参数：
参数 必选 类型及范围 说明
id true int64 要转发的微博ID。
status false string 添加的转发文本，必须做URLencode，内容不超过140个汉字，不填则默认为“转发微博”。
is_comment false int 是否在转发的同时发表评论，0：否、1：评论给当前微博、2：评论给原微博、3：都评论，默认为0
'''
self._checkAccessLimit('user_status')
with self.lock:
if len(content) > 139:
content = content[:139]
return self.client.post.statuses__repost(id=weibo_id, \
status=content, \
is_comment=is_comment)
def updateStatus(self, content, visible=0):
''' 发布新微博
一些参数：
参数 必选 类型及范围 说明
status true string 要发布的微博文本内容，必须做URLencode，内容不超过140个汉字。
visible false int 微博的可见性，0：所有人能看，1：仅自己可见，2：密友可见，3：指定分组可见，默认为0。
list_id false string 微博的保护投递指定分组ID，只有当visible参数为3时生效且必选。
lat false float 纬度，有效范围：-90.0到+90.0，+表示北纬，默认为0.0。
long false float 经度，有效范围：-180.0到+180.0，+表示东经，默认为0.0。
annotations false string 元数据，主要是为了方便第三方应用记录一些适合于自己使用的信息，每条微博可以包含一个或者多个元数据，必须以json字串的形式提交，字串长度不超过512个字符，具体内容可以自定。
'''
self._checkAccessLimit('user_status')
with self.lock:
if len(content) > 139:
content = content[:139]
return self.client.post.statuses__update(status=content, visible=visible)
def getShortUrl(self, url_long):
''' 获得短鏈 '''
self._checkAccessLimit()
with self.lock:
return self.client.get.short_url__shorten(url_long=url_long)['urls'][0]['url_short']
def getLongUrl(self, url_short):
''' 获得长鏈 '''
self._checkAccessLimit()
with self.lock:
return self.client.get.short_url__expand(url_short=url_short)['urls'][0]['url_long']
def setAccessToken(self, access_token, expires_in):
''' '''
with self.lock:
self.client.set_access_token(access_token, expires_in)
class WorkerDaemon(Daemon, MySQLConn, RedisConn):
def __init__(self,
pidfile,
workerid,
stdin=os.devnull,
stdout=os.devnull,
stderr=os.devnull):
# 初始化Daemon
super(WorkerDaemon, self).__init__(pidfile, stdin, stdout, stderr)
# 初始化MySQLConn
super(Daemon, self).__init__(config["mysql_config"]["host"],
config["mysql_config"]["port"],
config["mysql_config"]["db"],
config["mysql_config"]["username"],
config["mysql_config"]["password"])
# 初始化Redis
super(MySQLConn, self).__init__(
config["redis_config"]["host"],
config["redis_config"]["port"],
config["redis_config"]["db"],
)
self.worker_id = workerid
self.weibo_apps = []
self.delta_latlon = 0.001
# self.mysql_conn = MySQLdb.connect(host="localhost", user="******",
# passwd="admin", db="weibo_checkin")
# self.redis_conn = redis.Redis(host='localhost', port=6379, db=0,
# decode_responses=True)
self.doing_list = []
self.weibo_client = None
logging.info("Worker #%s inited." % workerid)
def run(self):
""" 运行worker """
self.get_weibo_token(
config["weibo_apps"][0]["app_key"],
config["weibo_apps"][0]["app_secret"],
config["weibo_apps"][0]["callback_url"],
config["weibo_apps"][0]["accounts"][0]["username"],
config["weibo_apps"][0]["accounts"][0]["password"],
)
# 如果程序意外退出，需要继续处理仍留在doing_list中的任务。
last_doing = self.redis_conn.lrange(
"poi_worker_" + str(self.worker_id) + "_doing_list", 0, -1)
last_doing = list(map(int, last_doing))
for doing in last_doing:
logging.info("poi last_doing found, taskid: %s" % doing)
self.execute_poi_task(doing)
# 监视新任务。
while True:
sys.stdout.write('.')
sys.stdout.flush()
self.mysql_conn.ping(True)
# 弹出poi_worker_1_todo_list。
todo = self.redis_conn.lpop("poi_worker_" + str(self.worker_id) +
"_todo_list")
if todo:
logging.info("poi_todo found, taskid: %s" % todo)
self.redis_conn.rpush(
"poi_worker_" + str(self.worker_id) + "_doing_list", todo)
self.doing_list.append(todo)
self.execute_poi_task(todo)
time.sleep(2)
def read_weibo_apps(self, _config):
self.weibo_apps = _config
def get_poi_task_x_worker_self(self, taskid):
task = self.redis_conn.hgetall("poi_task_" + str(taskid) + "_worker_" +
str(self.worker_id))
task = JsonDict(task)
task.cur_lat = float(task.cur_lat)
task.cur_lon = float(task.cur_lon)
task.max_lon = float(task.max_lon)
task.max_lat = float(task.max_lat)
task.min_lon = float(task.min_lon)
task.min_lat = float(task.min_lat)
task.progress = int(task.progress)
return task
def get_weibo_token(self, appkey, appsecret, url, username, password):
logging.info("preparing weibo OAuth2:")
logging.info("appkey: %s username: %s" % (appkey, username))
self.weibo_client = APIClient(app_key=appkey,
app_secret=appsecret,
redirect_uri=url)
code = WeiboLogin(username, password, appkey, url).get_code()
logging.info("code: %s" % code)
r = self.weibo_client.request_access_token(code)
self.weibo_client.set_access_token(r.access_token, r.expires_in)
logging.info("token: %s" % r.access_token)
def save_poi(self, poi, taskid):
sql = "SELECT `area_id` from `weibo_checkin_poitask` where `id` = ?"
res = self.mysql_select(sql, (taskid, ), 1, log=False)
areaid = int(res["area_id"])
sql = "SELECT `task_id` from `weibo_checkin_poi` where `poiid` = ?"
res = self.mysql_select(sql, (poi["poiid"], ), log=False)
if len(res) != 0:
return False
sql = "INSERT INTO `weibo_checkin_poi` " + \
"(`poiid`, `title`, `area_id`, `category_name`, `lon`, `lat`, " + \
"`icon`, `poi_pic`, `task_id`, `checkin_user_num`, `checkin_num`, `lat_baidu`, `lon_baidu`)" + \
"VALUES(?,?,?,?,?,?,?,?,?,?,?,null,null)"
args = (poi["poiid"], poi["title"], areaid, poi["category_name"],
float(poi["lon"]), float(poi["lat"]), poi["icon"],
poi["poi_pic"], taskid, int(poi["checkin_user_num"]),
int(poi["checkin_num"]))
res = self.mysql_execute(sql, args, log=False)
if res == 1:
self.redis_conn.hincrby(
"poi_task_" + str(taskid) + "_worker_" + str(self.worker_id),
"poi_add_count")
return True
return False
def save_checkin(self, checkin):
sql = "SELECT * from `weibo_checkin_checkin` where `mid` = ?"
logging.info(checkin)
res = self.mysql_select(sql, (checkin["mid"], ))
if len(res) != 0:
return False
time_obj = time.strptime(checkin["created_at"],
"%a %b %d %H:%M:%S %z %Y")
time_str = time.strftime("%Y-%m-%d %H:%M:%S", time_obj)
sql = "INSERT INTO `weibo_checkin_checkin` " + \
"(`mid`, `text`, `created_at`, `user_name`, `poi_id`) " + \
"VALUES(?,?,?,?,?)"
try:
args = (checkin["mid"], checkin["text"][0:20], time_str,
checkin["user"]["name"],
checkin["annotations"][0]["place"]["poiid"])
except KeyError:
# 比如，有些deleted为1的条目没有checkin["user']
return False
try:
res = self.mysql_execute(sql, args)
except OperationalError:
logging.warning("incorrect charater found in: %s" %
checkin["text"])
args = (checkin["mid"], "[该微博包括特殊字符，不能存储。]", time_str,
checkin["user"]["name"],
checkin["annotations"][0]["place"]["poiid"])
res = self.mysql_execute(sql, args)
if res == 1:
return True
return False
def get_checkins_at(self, poiid):
if self.weibo_client.is_expires():
self.get_weibo_token(
config["weibo_apps"][0]["app_key"],
config["weibo_apps"][0]["app_secret"],
config["weibo_apps"][0]["callback_url"],
config["weibo_apps"][0]["accounts"][0]["username"],
config["weibo_apps"][0]["accounts"][0]["password"],
)
page = 1
if self.redis_conn.exists("checkin_task_" + poiid + "_page"):
page = int(self.redis_conn.get("checkin_task_" + poiid + "_page"))
res = self.weibo_client.place.poi_timeline.get(poiid=poiid,
page=page,
count=50)
while True:
if not res:
break
if not res["statuses"]:
break
page_add = 0
last_time = None
if isinstance(res["statuses"], list):
for item in res["statuses"]:
last_time = time.strptime(item["created_at"],
"%a %b %d %H:%M:%S %z %Y")
if self.save_checkin(item):
page_add += 1
else:
for key in res["statuses"].keys():
last_time = time.strptime(
res["statuses"][key]["created_at"],
"%a %b %d %H:%M:%S %z %Y")
if self.save_checkin(res["statuses"][key]):
page_add += 1
logging.info("checkin page: %s, total: %s, add: %s" %
(page, len(res["statuses"]), page_add))
# 只取最近一个月的。
if time.mktime(last_time) + 3600 * 24 * 30 < time.mktime(
time.localtime()):
break
page += 1
self.redis_conn.set("checkin_task_" + poiid + "_page", page)
if page > 100:
break
if self.weibo_client.is_expires():
self.get_weibo_token(
config["weibo_apps"][0]["app_key"],
config["weibo_apps"][0]["app_secret"],
config["weibo_apps"][0]["callback_url"],
config["weibo_apps"][0]["accounts"][0]["username"],
config["weibo_apps"][0]["accounts"][0]["password"],
)
time.sleep(2)
res = self.weibo_client.place.poi_timeline.get(poiid=poiid,
page=page,
count=50)
self.redis_conn.delete("checkin_task_" + poiid + "_page")
def get_pois_at(self, lon, lat, taskid):
# time.sleep(2)
# return
if self.weibo_client.is_expires():
self.get_weibo_token(
config["weibo_apps"][0]["app_key"],
config["weibo_apps"][0]["app_secret"],
config["weibo_apps"][0]["callback_url"],
config["weibo_apps"][0]["accounts"][0]["username"],
config["weibo_apps"][0]["accounts"][0]["password"],
)
page = 1
if self.redis_conn.exists("poi_task_" + taskid + "_page"):
page = int(self.redis_conn.get("poi_task_" + taskid + "_page"))
res = self.weibo_client.place.nearby.pois.get(lat=lat,
long=lon,
page=page,
range=100,
count=50)
while res:
page_add = 0
cur_poiid = ""
if self.redis_conn.exists("poi_task_" + taskid + "_poiid"):
cur_poiid = self.redis_conn.get("poi_task_" + taskid +
"_poiid")
for item in res["pois"]:
if self.save_poi(item, taskid):
page_add += 1
self.get_checkins_at(item["poiid"])
self.redis_conn.set("poi_task_" + taskid + "_poiid",
item["poiid"])
else:
# 获取签到信息。
if cur_poiid == "":
continue
else:
if cur_poiid == item["poiid"]:
self.get_checkins_at(item["poiid"])
cur_poiid = ""
self.redis_conn.delete("poi_task_" + taskid + "_poiid")
logging.info("lon: %s, lat: %s, page: %s, total: %s, add: %s" %
(lon, lat, page, len(res["pois"]), page_add))
page += 1
self.redis_conn.set("poi_task_" + taskid + "_page", page)
if self.weibo_client.is_expires():
self.get_weibo_token(
config["weibo_apps"][0]["app_key"],
config["weibo_apps"][0]["app_secret"],
config["weibo_apps"][0]["callback_url"],
config["weibo_apps"][0]["accounts"][0]["username"],
config["weibo_apps"][0]["accounts"][0]["password"],
)
time.sleep(2)
res = self.weibo_client.place.nearby.pois.get(lat=lat,
long=lon,
page=page,
range=100,
count=50)
self.redis_conn.delete("poi_task_" + taskid + "_page")
def execute_poi_task(self, taskid):
""" 开始/继续一个任务 """
try:
pid = os.fork()
if pid == 0: # 在子进程中进行任务。
# 得到当前任务进行的信息。
# 只要没有检测到暂停指令，就继续进行任务。
logging.info("[%s]execute poi task #%s." %
(os.getpid(), taskid))
task = self.get_poi_task_x_worker_self(taskid)
lon_total = int(
(task.max_lon - task.min_lon) / self.delta_latlon) + 1
lat_total = int(
(task.max_lat - task.min_lat) / self.delta_latlon) + 1
logging.info("lon_total: %s, lat_total: %s" %
(lon_total, lat_total))
while not self.redis_conn.exists("poi_" + str(taskid) +
"_to_pause"):
# 任务完成
if task.cur_lat > task.max_lat:
self.redis_conn.hset(
"poi_task_" + str(taskid) + "_worker_" +
str(self.worker_id), "progress", 100)
self.redis_conn.lpop("poi_worker_" +
str(self.worker_id) +
"_doing_list")
logging.info("poi task #%s finish." % taskid)
sys.exit()
try:
# 下载数据
self.get_pois_at(task.cur_lon, task.cur_lat, taskid)
# 计算progress
lon = round((task.cur_lon - task.min_lon) /
self.delta_latlon) + 1
lat = round((task.cur_lat - task.min_lat) /
self.delta_latlon) + 1
progress = round((lon_total * (lat - 1) + lon) /
(lat_total * lon_total) * 100)
# if progress > 20:
# raise WeiboLoginError(0, "test error. (progress > 20)")
self.redis_conn.hset(
"poi_task_" + str(taskid) + "_worker_" +
str(self.worker_id), "progress", progress)
logging.info(
"cur_lon: %s, cur_lat: %s, cur_progress: %s" %
(task.cur_lon, task.cur_lat, progress))
# 设置新的cur_lat、cur_lon
new_lat = task.cur_lat
new_lon = task.cur_lon + self.delta_latlon
if new_lon > task.max_lon:
new_lon = task.min_lon
new_lat = task.cur_lat + self.delta_latlon
self.redis_conn.hset(
"poi_task_" + str(taskid) + "_worker_" +
str(self.worker_id), "cur_lon", new_lon)
self.redis_conn.hset(
"poi_task_" + str(taskid) + "_worker_" +
str(self.worker_id), "cur_lat", new_lat)
task = self.get_poi_task_x_worker_self(taskid)
except WeiboLoginError as e:
errmsg = str(e)
self.redis_conn.lpop("poi_worker_" +
str(self.worker_id) +
"_doing_list")
logging.info("poi task #%s error: %s" %
(taskid, errmsg))
self.redis_conn.hset(
"poi_task_" + str(taskid) + "_worker_" +
str(self.worker_id), "errormsg", errmsg)
sys.exit()
except BaseException:
# 任务出错
errmsg = "some reason i don't know"
self.redis_conn.lpop("poi_worker_" +
str(self.worker_id) +
"_doing_list")
logging.info("poi task #%s error: %s" %
(taskid, errmsg))
self.redis_conn.hset(
"poi_task_" + str(taskid) + "_worker_" +
str(self.worker_id), "errormsg", errmsg)
raise
time.sleep(2) # 每次网络请求的间隔
# 由于用户主动暂停而终止：
self.redis_conn.lpop("poi_worker_" + str(self.worker_id) +
"_doing_list")
self.redis_conn.hset(
"poi_task_" + str(taskid) + "_worker_" +
str(self.worker_id), "errormsg", "用户暂停")
logging.info("task #%s paused." % taskid)
# todo: 更新cur_lat和cur_lon
# todo: 这里的doing_list只能容纳一个元素
sys.exit()
except OSError as err:
sys.stderr.write('fork failed: {0}\n'.format(err))
sys.exit(1)
__author__ = 'zice' from weibo import APIClient APP_KEY = '2038546250' APP_SECRET = '24bb05a6fb21eadf36713d75f2fe2b77' CALLBACK_URL = 'http://www.example.com/callback' code = '8067a19c36a182b6643d77825ccc3ab4' client = APIClient(app_key=APP_KEY, app_secret=APP_SECRET, redirect_uri=CALLBACK_URL) r = client.request_access_token(code) access_token = r.access_token expires_in = r.expires_in print access_token print expires_in client.set_access_token(access_token, expires_in) s = client.is_expires() print s
__author__ = 'zice'
from weibo import APIClient
APP_KEY = '2038546250'
APP_SECRET = '24bb05a6fb21eadf36713d75f2fe2b77'
CALLBACK_URL = 'http://www.example.com/callback'
code = '52b9758e5abe877e1cb0c337b4ca0fa7'
client = APIClient(app_key=APP_KEY,
app_secret=APP_SECRET,
redirect_uri=CALLBACK_URL)
r = client.request_access_token(code)
access_token = r.access_token
expires_in = r.expires_in
print access_token
print expires_in
client.set_access_token(access_token, expires_in)
s = client.is_expires()
print s
