def main(): """docstring for main""" #set system default encoding to utf-8 to avoid encoding problems reload(sys) sys.setdefaultencoding( "utf-8" ) #load channel configurations channels = json.load(open('conf/channel.json')) #find one account rr = SNSPocket() for c in channels: rr.add_channel(c) if rr is None: print "cannot find one renren platform in channel.json" return else: rr.load_config() rr.auth() #load record to avoid repeated reply try: sIDs = json.load(open('statusID.json')) except IOError, e: if e.errno == 2: #no such file sIDs = {} else: raise e
def _do_get_pocket(_sp_cache=[]): if _sp_cache: return _sp_cache[0] sp = SNSPocket() sp.load_config(CONF_ABS_PATH) sp.auth() _sp_cache.append(sp) return sp
def crawl_one(tag): sp = SNSPocket() sp.load_config(fn_channel=tag+'/channel.json') status_list = mongo_client[tag].status_list sl = sp.home_timeline() for s in sl: try: r = s.raw #print r clear_urls(r) if 'retweeted_status' in r: clear_urls(r['retweeted_status']) r['_id'] = r['id'] r['time'] = s.parsed['time'] ret = status_list.insert(r) print ret except Exception as e: print str(e)
def begin_user_thread(user): user_sp = {} ap = AuthProxy() for (k,v) in user.items(): p = build_user_dir(k) os.chdir("." + delimeter + "User" + delimeter + str(k)) sp = SNSPocket() sp.load_config(fn_channel= p + delimeter + "conf" + delimeter + "channel.json",fn_pocket= p + delimeter + "conf" + delimeter + "pocket.json") for c in sp.values(): c.request_url = lambda url: ap.request_url(url) c.fetch_code = lambda : ap.fetch_code() c.auth() q = SRFEQueue(sp) q.connect() q.refresh_tags() ith = InputThread(q, sp, k) ith.daemon=True ith.start() user_sp[str(k)] = {"sp" : sp, "ap" : ap, "q" : q} os.chdir(".." + delimeter + ".." + delimeter) return user_sp
def register(): username = request.forms.get('username') password = request.forms.get('password') v = {} v["username"] = username v['password'] = password v["token"] = hashlib.new('md5', username + password + str(time.time())).hexdigest() v["generate_time"] = time.time() cur = con.cursor() cur.execute("INSERT INTO user VALUES(null, ?, ?, ?, ?)", (v["username"], v["password"], v["token"], v["generate_time"])) con.commit(); r = cur.execute("SELECT id FROM user WHERE username = ?", (v["username"], )) for m in r: k = m[0] user[str(k)] = v build_user_dir(str(m[0])) p = build_user_dir(k) os.chdir("." + delimeter + "User" + delimeter + str(k)) ap = AuthProxy() sp = SNSPocket() sp.load_config(fn_channel= p + delimeter + "conf" + delimeter + "channel.json",fn_pocket= p + delimeter + "conf" + delimeter + "pocket.json") for c in sp.values(): c.request_url = lambda url: ap.request_url(url) c.fetch_code = lambda : ap.fetch_code() c.auth() q = SRFEQueue(sp) q.connect() q.refresh_tags() ith = InputThread(q, sp, k) ith.daemon=True ith.start() user_sp[str(k)] = {"sp" : sp, "ap" : ap, "q" : q} os.chdir("../../") response.set_cookie("token", v["token"], httponly="on", max_age = TOKEN_VALID_PERIOD) return "Reg success"
# -*- coding: utf-8 -*- from snsapi.snspocket import SNSPocket from datetime import datetime import time TEXTS = ['凌晨好', '清晨好', '早上好', '下午好', '傍晚好', '晚上好'] #URL = 'https://github.com/hupili/snsapi/tree/master/app/clock' URL = 'http://t.cn/zj1VSdV' AD = '10行写个跨平台的钟:%s' % URL sp = SNSPocket() # SNSPocket 是一个承载各种SNS的容器 sp.load_config() # 如名 sp.auth() # 批量授权(如果已授权,读取授权信息) while True: h, m = datetime.now().hour, datetime.now().minute # 获取当前小时和分钟 if m == 0: # 每小时0分钟的时候发状态 t = '%s -- 0x%X点钟, %s。( %s )' % ('烫' * h, h, TEXTS[h / 4], AD) # 构造钟的报时文字 print t sp.update(t) # 发一条新状态 time.sleep(60) # 睡一分钟
# -*- coding: utf-8 -*- from snsapi.snspocket import SNSPocket from datetime import datetime import time TEXTS = ['凌晨好', '清晨好', '早上好', '下午好', '傍晚好', '晚上好'] #URL = 'https://github.com/hupili/snsapi/tree/master/app/clock' URL = 'http://t.cn/zj1VSdV' AD = '10行写个跨平台的钟:%s' % URL sp = SNSPocket() # SNSPocket 是一个承载各种SNS的容器 sp.load_config() # 如名 sp.auth() # 批量授权(如果已授权,读取授权信息) while True: h, m = datetime.now().hour, datetime.now().minute # 获取当前小时和分钟 if m == 0: # 每小时0分钟的时候发状态 t = '%s -- 0x%X点钟, %s。( %s )' % ('烫' * h, h, TEXTS[h / 4], AD ) # 构造钟的报时文字 print t sp.update(t) # 发一条新状态 time.sleep(60) # 睡一分钟
# -*- coding: utf-8 -*- ''' Read timeline from all configured channels docstring placeholder ''' from snsapi.snspocket import SNSPocket from snsapi.utils import console_input,console_output if __name__ == "__main__": sp = SNSPocket() sp.load_config() sp.auth() sl = sp.home_timeline() print sl
''' A wrapper function to deal with user input from console. String input from console is in console encoding. We must first cast it to unicode, which is the standard across SNSAPI. ''' if isinstance(t, str): return sp.update(console_input(t), *al, **ad) elif isinstance(t, snstype.Message): return sp.update(t, *al, **ad) else: logger.warning("unknown type: %s", type(t)) lc = load_config = lambda *al, **ad : sp.load_config(*al, **ad) sc = save_config = lambda *al, **ad : sp.save_config(*al, **ad) lsc = list_channel = lambda *al, **ad : sp.list_channel(*al, **ad) lsp = list_platform = lambda *al, **ad : sp.list_platform(*al, **ad) newc = new_channel = lambda *al, **ad : sp.new_channel(*al, **ad) addc = add_channel = lambda *al, **ad : sp.add_channel(*al, **ad) clc = clear_channel = lambda *al, **ad : sp.clear_channel(*al, **ad) auth = lambda *al, **ad : sp.auth(*al, **ad) ht = home_timeline = lambda *al, **ad : sp.home_timeline(*al, **ad) up = update = lambda t, *al, **ad : update_from_console(t, *al, **ad) re = reply = lambda m, t, *al, **ad : sp.reply(m, console_input(t), *al, **ad) fwd = forward = lambda m, t, *al, **ad : sp.forward(m, console_input(t), *al, **ad) #==== documentation ==== helpdoc = \
Read timeline from all configured channels and reply one docstring placeholder ''' from snsapi.snspocket import SNSPocket from snsapi.utils import console_input, console_output if __name__ == "__main__": ''' QQ weibo may fail sometimes, even with same input. May be the invoking frequency limit. Sina weibo is better, and more stable. ''' sp = SNSPocket() sp.load_config() sp.auth() status_list = sp.home_timeline() print "==== read messages from all channels ====" no = 0 for s in status_list: print "--No. %d --" % no s.show() no = no + 1 print "==== try to reply one ===="
class WeiboAutomator(object): '''Wrap common operations with rate limit facility ''' # Most buckets are derived from Sina's offcial description [1]. # The additional one 'wauto_snsapi' limits SNSAPI request rate globally # (avoid lower layer failure). # # Ref: # * [1] http://open.weibo.com/wiki/Rate-limiting SINA_BUCKETS = [ ('wauto_snsapi', LeakyBucket(1, 0, 0.5)), ('ip.hour.test_auth', cal_bucket(1000, 60*60)), ('user.hour.test_auth.total', cal_bucket(150, 60*60)), ('user.hour.test_auth.update', cal_bucket(30, 60*60)), ('user.hour.test_auth.reply', cal_bucket(60, 60*60)), ('user.hour.test_auth.follow', cal_bucket(60, 60*60)), ('user.day.test_auth.follow', cal_bucket(100, 60*60*24)), ] POLICY_GROUP = {} POLICY_GROUP['general'] = { 'wauto_snsapi': 1, 'ip.hour.test_auth': 1, 'user.hour.test_auth.total': 1 } POLICY_GROUP['update'] = dict(POLICY_GROUP['general'], **{'user.hour.test_auth.update': 1}) POLICY_GROUP['reply'] = dict(POLICY_GROUP['general'], **{'user.hour.test_auth.reply': 1}) POLICY_GROUP['follow'] = dict(POLICY_GROUP['general'], **{'user.hour.test_auth.follow': 1, 'user.day.test_auth.follow': 1}) _log = lambda x: logger.debug('ret: %s', x) def __init__(self): super(WeiboAutomator, self).__init__() self.sp = SNSPocket() self.sp.load_config() self.sp.auth() # assign 'channel_name' as automator self.weibo = self.sp['automator'] self.rlq = RateLimitQueue() map(lambda t: self.rlq.add_bucket(t[0], t[1]), self.SINA_BUCKETS) # This implementation of dumps and loads are too simple. # They do not work with some callback functions. # I switch to 'dill' #def dumps(self): # r = copy.deepcopy(self.rlq) # for t in r._tasks: # # First arg should be 'self' if do not operate our RLQ directly. # t.args = list(t.args) # t.args.pop(0) # t.func = marshal.dumps(t.func.func_code) # t.callback = marshal.dumps(t.callback.func_code) # return pickle.dumps(r) #def loads(self, s): # r = pickle.loads(s) # for t in r._tasks: # t.args.insert(0, self) # t.args = tuple(t.args) # code_func = marshal.loads(t.func) # t.func = types.FunctionType(code_func, globals()) # code_callback = marshal.loads(t.callback) # t.callback = types.FunctionType(code_callback, globals()) # self.rlq = r #Original 'loads' 2 # #self.rlq._buckets = r._buckets # #for t in r._tasks: # # code = marshal.loads(t.callback) # # t.callback = types.FunctionType(code, globals()) # # t.args.insert(0, self) # # t.args = tuple(t.args) # # t.kwargs['callback'] = t.callback # # f = getattr(WeiboAutomator, t.func) # # # Execute the wrapped class method again to insert task # # f(*t.args, **t.kwargs) def run(self): return self.rlq.run() def clear_tasks(self): return self.rlq.clear_tasks() def _tounicode(self, text): if isinstance(text, unicode): return text else: return text.decode('utf-8') def get_uid(self): if hasattr(self, '_uid'): return self._uid else: ret = self.weibo.weibo_request('account/get_uid', 'GET', {}) return ret['uid'] uid = property(get_uid) def rate_limit_status(self): ret = self.weibo.weibo_request('account/rate_limit_status', 'GET', {}) return ret @rate_limit(buckets=POLICY_GROUP['follow'], callback=_log) def follow(self, uid): ret = self.weibo.weibo_request('friendships/create', 'POST', {'uid': uid}) return ret @rate_limit(buckets=POLICY_GROUP['follow'], callback=_log) def follow_by_name(self, screen_name): ret = self.weibo.weibo_request('friendships/create', 'POST', {'screen_name': screen_name}) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log, priority=_wauto_conf['priority']['home_timeline']) def home_timeline(self, count=20): return self.weibo.home_timeline(count) @rate_limit(buckets=POLICY_GROUP['update'], callback=_log, priority=_wauto_conf['priority']['update']) def update(self, text): return self.weibo.update(self._tounicode(text)) @rate_limit(buckets=POLICY_GROUP['reply'], callback=_log, priority=_wauto_conf['priority']['reply']) def reply(self, status, text): if isinstance(status, snsapi.snstype.Message): statusID = status.ID else: statusID = status return self.weibo.reply(statusID, self._tounicode(text)) @rate_limit(buckets=POLICY_GROUP['general'], callback=_log, priority=_wauto_conf['priority']['forward']) def forward(self, status, text): return self.weibo.forward(status, self._tounicode(text)) @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def show(self, uid = None, screen_name = None): params = {} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid ret = self.weibo.weibo_request('users/show', 'GET', params) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def domain_show(self, url): '''Lookup user by personal url. We will match and remove common weibo prefix. :param url: e.g. 'http://weibo.com/xiena' --> url='xiena' ''' import re pattern = re.compile('^http:\/\/.*weibo.com\/') url = re.sub(pattern, '', url) ret = self.weibo.weibo_request('users/domain_show', 'GET', {'domain': url}) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def get_friends(self, uid=None, screen_name=None, count=200, cursor=None): params = {'count': count} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid if not cursor is None: params['cursor'] = cursor ret = self.weibo.weibo_request('friendships/friends', 'GET', params) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def get_friends_ids(self, uid=None, screen_name=None, count=5000, cursor=None): params = {'count': count} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid if not cursor is None: params['cursor'] = cursor ret = self.weibo.weibo_request('friendships/friends/ids', 'GET', params) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def get_followers(self, uid=None, screen_name=None, count=200, cursor=None): params = {'count': count} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid if not cursor is None: params['cursor'] = cursor ret = self.weibo.weibo_request('friendships/followers', 'GET', params) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def get_followers_ids(self, uid=None, screen_name=None, count=5000, cursor=None): params = {'count': count} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid if not cursor is None: params['cursor'] = cursor ret = self.weibo.weibo_request('friendships/followers/ids', 'GET', params) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def get_followers_active(self, uid=None, screen_name=None, count=200, cursor=None): params = {'count': count} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid if not cursor is None: params['cursor'] = cursor ret = self.weibo.weibo_request('friendships/followers/active', 'GET', params) return ret # This Api is only for advanced app permission @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def search_topics(self, q, count=50, page=None): params = {'count': count} if not page is None: params['page'] = page ret = self.weibo.weibo_request('search/topics', 'GET', params) return ret
A wrapper function to deal with user input from console. String input from console is in console encoding. We must first cast it to unicode, which is the standard across SNSAPI. ''' if isinstance(t, str): return sp.update(console_input(t), *al, **ad) elif isinstance(t, snstype.Message): return sp.update(t, *al, **ad) else: logger.warning("unknown type: %s", type(t)) lc = load_config = lambda *al, **ad: sp.load_config(*al, **ad) sc = save_config = lambda *al, **ad: sp.save_config(*al, **ad) lsc = list_channel = lambda *al, **ad: sp.list_channel(*al, **ad) lsp = list_platform = lambda *al, **ad: sp.list_platform(*al, **ad) newc = new_channel = lambda *al, **ad: sp.new_channel(*al, **ad) addc = add_channel = lambda *al, **ad: sp.add_channel(*al, **ad) clc = clear_channel = lambda *al, **ad: sp.clear_channel(*al, **ad) auth = lambda *al, **ad: sp.auth(*al, **ad) ht = home_timeline = lambda *al, **ad: sp.home_timeline(*al, **ad) up = update = lambda t, *al, **ad: update_from_console(t, *al, **ad) re = reply = lambda m, t, *al, **ad: sp.reply(m, console_input(t), *al, **ad) fwd = forward = lambda m, t, *al, **ad: sp.forward(m, console_input(t), *al, ** ad) #==== documentation ====
class WeiboAutomator(object): '''Wrap common operations with rate limit facility ''' # Most buckets are derived from Sina's offcial description [1]. # The additional one 'wauto_snsapi' limits SNSAPI request rate globally # (avoid lower layer failure). # # Ref: # * [1] http://open.weibo.com/wiki/Rate-limiting SINA_BUCKETS = [ ('wauto_snsapi', LeakyBucket(1, 0, 0.5)), ('ip.hour.test_auth', cal_bucket(1000, 60 * 60)), ('user.hour.test_auth.total', cal_bucket(150, 60 * 60)), ('user.hour.test_auth.update', cal_bucket(30, 60 * 60)), ('user.hour.test_auth.reply', cal_bucket(60, 60 * 60)), ('user.hour.test_auth.follow', cal_bucket(60, 60 * 60)), ('user.day.test_auth.follow', cal_bucket(100, 60 * 60 * 24)), ] POLICY_GROUP = {} POLICY_GROUP['general'] = { 'wauto_snsapi': 1, 'ip.hour.test_auth': 1, 'user.hour.test_auth.total': 1 } POLICY_GROUP['update'] = dict(POLICY_GROUP['general'], **{'user.hour.test_auth.update': 1}) POLICY_GROUP['reply'] = dict(POLICY_GROUP['general'], **{'user.hour.test_auth.reply': 1}) POLICY_GROUP['follow'] = dict( POLICY_GROUP['general'], **{ 'user.hour.test_auth.follow': 1, 'user.day.test_auth.follow': 1 }) _log = lambda x: logger.debug('ret: %s', x) def __init__(self): super(WeiboAutomator, self).__init__() self.sp = SNSPocket() self.sp.load_config() self.sp.auth() # assign 'channel_name' as automator self.weibo = self.sp['automator'] self.rlq = RateLimitQueue() map(lambda t: self.rlq.add_bucket(t[0], t[1]), self.SINA_BUCKETS) # This implementation of dumps and loads are too simple. # They do not work with some callback functions. # I switch to 'dill' #def dumps(self): # r = copy.deepcopy(self.rlq) # for t in r._tasks: # # First arg should be 'self' if do not operate our RLQ directly. # t.args = list(t.args) # t.args.pop(0) # t.func = marshal.dumps(t.func.func_code) # t.callback = marshal.dumps(t.callback.func_code) # return pickle.dumps(r) #def loads(self, s): # r = pickle.loads(s) # for t in r._tasks: # t.args.insert(0, self) # t.args = tuple(t.args) # code_func = marshal.loads(t.func) # t.func = types.FunctionType(code_func, globals()) # code_callback = marshal.loads(t.callback) # t.callback = types.FunctionType(code_callback, globals()) # self.rlq = r #Original 'loads' 2 # #self.rlq._buckets = r._buckets # #for t in r._tasks: # # code = marshal.loads(t.callback) # # t.callback = types.FunctionType(code, globals()) # # t.args.insert(0, self) # # t.args = tuple(t.args) # # t.kwargs['callback'] = t.callback # # f = getattr(WeiboAutomator, t.func) # # # Execute the wrapped class method again to insert task # # f(*t.args, **t.kwargs) def run(self): return self.rlq.run() def clear_tasks(self): return self.rlq.clear_tasks() def _tounicode(self, text): if isinstance(text, unicode): return text else: return text.decode('utf-8') def get_uid(self): if hasattr(self, '_uid'): return self._uid else: ret = self.weibo.weibo_request('account/get_uid', 'GET', {}) return ret['uid'] uid = property(get_uid) def rate_limit_status(self): ret = self.weibo.weibo_request('account/rate_limit_status', 'GET', {}) return ret @rate_limit(buckets=POLICY_GROUP['follow'], callback=_log) def follow(self, uid): ret = self.weibo.weibo_request('friendships/create', 'POST', {'uid': uid}) return ret @rate_limit(buckets=POLICY_GROUP['follow'], callback=_log) def follow_by_name(self, screen_name): ret = self.weibo.weibo_request('friendships/create', 'POST', {'screen_name': screen_name}) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log, priority=_wauto_conf['priority']['home_timeline']) def home_timeline(self, count=20): return self.weibo.home_timeline(count) @rate_limit(buckets=POLICY_GROUP['update'], callback=_log, priority=_wauto_conf['priority']['update']) def update(self, text): return self.weibo.update(self._tounicode(text)) @rate_limit(buckets=POLICY_GROUP['reply'], callback=_log, priority=_wauto_conf['priority']['reply']) def reply(self, status, text): if isinstance(status, snsapi.snstype.Message): statusID = status.ID else: statusID = status return self.weibo.reply(statusID, self._tounicode(text)) @rate_limit(buckets=POLICY_GROUP['general'], callback=_log, priority=_wauto_conf['priority']['forward']) def forward(self, status, text): return self.weibo.forward(status, self._tounicode(text)) @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def show(self, uid=None, screen_name=None): params = {} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid ret = self.weibo.weibo_request('users/show', 'GET', params) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def domain_show(self, url): '''Lookup user by personal url. We will match and remove common weibo prefix. :param url: e.g. 'http://weibo.com/xiena' --> url='xiena' ''' import re pattern = re.compile('^http:\/\/.*weibo.com\/') url = re.sub(pattern, '', url) ret = self.weibo.weibo_request('users/domain_show', 'GET', {'domain': url}) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def get_friends(self, uid=None, screen_name=None, count=200, cursor=None): params = {'count': count} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid if not cursor is None: params['cursor'] = cursor ret = self.weibo.weibo_request('friendships/friends', 'GET', params) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def get_friends_ids(self, uid=None, screen_name=None, count=5000, cursor=None): params = {'count': count} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid if not cursor is None: params['cursor'] = cursor ret = self.weibo.weibo_request('friendships/friends/ids', 'GET', params) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def get_followers(self, uid=None, screen_name=None, count=200, cursor=None): params = {'count': count} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid if not cursor is None: params['cursor'] = cursor ret = self.weibo.weibo_request('friendships/followers', 'GET', params) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def get_followers_ids(self, uid=None, screen_name=None, count=5000, cursor=None): params = {'count': count} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid if not cursor is None: params['cursor'] = cursor ret = self.weibo.weibo_request('friendships/followers/ids', 'GET', params) return ret @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def get_followers_active(self, uid=None, screen_name=None, count=200, cursor=None): params = {'count': count} if not uid is None: params['uid'] = uid elif not screen_name is None: params['screen_name'] = screen_name else: params['uid'] = self.uid if not cursor is None: params['cursor'] = cursor ret = self.weibo.weibo_request('friendships/followers/active', 'GET', params) return ret # This Api is only for advanced app permission @rate_limit(buckets=POLICY_GROUP['general'], callback=_log) def search_topics(self, q, count=50, page=None): params = {'count': count} if not page is None: params['page'] = page ret = self.weibo.weibo_request('search/topics', 'GET', params) return ret