def _parse(self, dct): if 'deleted' in dct and dct['deleted']: logger.debug("This is a deleted message %s of SinaWeiboStatusMessage", dct["id"]) self.parsed.time = "unknown" self.parsed.username = "******" self.parsed.userid = "unknown" self.parsed.text = "unknown" self.deleted = True return self.ID.id = dct["id"] self.parsed.time = utils.str2utc(dct["created_at"]) self.parsed.username = dct['user']['name'] self.parsed.userid = dct['user']['id'] self.parsed.reposts_count = dct['reposts_count'] self.parsed.comments_count = dct['comments_count'] if 'retweeted_status' in dct: self.parsed.username_orig = "unknown" try: self.parsed.username_orig = dct['retweeted_status']['user']['name'] except KeyError: logger.warning('KeyError when parsing SinaWeiboStatus. May be deleted original message') self.parsed.text_orig = dct['retweeted_status']['text'] self.parsed.text_trace = dct['text'] self.parsed.text = self.parsed.text_trace \ + " || " + "@" + self.parsed.username_orig \ + " : " + self.parsed.text_orig else: self.parsed.text_orig = dct['text'] self.parsed.text_trace = None self.parsed.text = self.parsed.text_orig
def reply(self, mID, text): ''' Reply a renren blog :param mID: MessageID object :param text: string, the reply message :return: success or not ''' if mID.user_type == 'user': owner_key = 'uid' owner_value = mID.source_user_id else: # 'page' owner_key = 'page_id' owner_value = mID.source_page_id api_params = { 'method': 'blog.addComment', 'content': text, 'id': mID.blog_id, owner_key: owner_value } logger.debug('request parameters: %s', api_params) try: ret = self.renren_request(api_params) if 'result' in ret and ret['result'] == 1: logger.info("Reply '%s' to status '%s' succeed", text, mID) return True except Exception, e: logger.warning("Catch Exception %s", e)
def update(self, text, title=None): ''' Post a blog :param text: Blog post body. :param title: Blog post title. (optional) :return: success or not ''' if title is None: title = self._cat(20, [(text, 1)]) api_params = {'method': 'blog.addBlog', 'content': text, 'title': title} try: ret = self.renren_request(api_params) logger.debug("response: %s", ret) #TODO: # Preserve the id for further use? # Return it as multi-return-value? if 'id' in ret: logger.info("Update status '%s' on '%s' succeed", text, self.jsonconf.channel_name) return True except Exception, e: logger.warning("Catch Exception %s", e)
def reply(self, mID, text): ''' Reply a renren blog :param mID: MessageID object :param text: string, the reply message :return: success or not ''' if mID.user_type == 'user': owner_key = 'uid' owner_value = mID.source_user_id else: # 'page' owner_key = 'page_id' owner_value = mID.source_page_id api_params = {'method': 'blog.addComment', 'content': text, 'id': mID.blog_id, owner_key: owner_value} logger.debug('request parameters: %s', api_params) try: ret = self.renren_request(api_params) if 'result' in ret and ret['result'] == 1: logger.info("Reply '%s' to status '%s' succeed", text, mID) return True except Exception, e: logger.warning("Catch Exception %s", e)
def update(self, text, title=None): ''' Post a blog :param text: Blog post body. :param title: Blog post title. (optional) :return: success or not ''' if title is None: title = self._cat(20, [(text, 1)]) api_params = { 'method': 'blog.addBlog', 'content': text, 'title': title } try: ret = self.renren_request(api_params) logger.debug("response: %s", ret) #TODO: # Preserve the id for further use? # Return it as multi-return-value? if 'id' in ret: logger.info("Update status '%s' on '%s' succeed", text, self.jsonconf.channel_name) return True except Exception, e: logger.warning("Catch Exception %s", e)
def _fetch_code_local_username_password(self): try: login_username = self.auth_info.login_username login_password = self.auth_info.login_password app_key = self.jsonconf.app_key app_secret = self.jsonconf.app_secret callback_url = self.auth_info.callback_url referer_url = self._last_requested_url postdata = {"client_id": app_key, "redirect_uri": callback_url, "userId": login_username, "passwd": login_password, "isLoginSina": "0", "action": "submit", "response_type": "code", } headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0", "Host": "api.weibo.com", "Referer": referer_url } auth_url = "https://api.weibo.com/oauth2/authorize" # auth_url = self.auth_info.auth_url resp_url = self._http_post(auth_url, data=postdata, headers=headers, json_parse=False).url logger.debug("response URL from local post: %s", resp_url) return resp_url except Exception, e: logger.warning("Catch exception: %s", e)
def reply(self, statusID, text): ''' docstring placeholder ''' #"""reply status #@param status: StatusID object #@param text: string, the reply message #@return: success or not #""" api_params = dict(method = "share.addComment", content = text, \ share_id = statusID.status_id, user_id = statusID.source_user_id) try: ret = self.renren_request(api_params) logger.debug("Reply to status '%s' return: %s", statusID, ret) if 'result' in ret and ret['result'] == 1: logger.info("Reply '%s' to status '%s' succeed", text, statusID) return True else: return False except Exception, e: logger.warning("Reply failed: %s", e)
def request_url(self, url): if self.auth_info.cmd_request_url == "(webbrowser)": self.open_brower(url) elif self.auth_info.cmd_request_url == "(console_output)": utils.console_output(url) elif self.auth_info.cmd_request_url == "(local_webserver)+(webbrowser)": host = self.auth_info.host port = self.auth_info.port from third.server import ClientRedirectServer from third.server import ClientRedirectHandler import socket try: self.httpd = ClientRedirectServer((host, port), ClientRedirectHandler) self.open_brower(url) except socket.error: raise snserror.auth else: self.__last_request_time = self.time() cmd = "%s '%s'" % (self.auth_info.cmd_request_url, url) logger.debug("request_url command is: %s", cmd) res = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout.read().rstrip() logger.debug("request_url result is: %s", res) return
def reply(self, statusID, text): ''' docstring placeholder ''' """reply status * parameter status: StatusID object * paramter text: string, the reply message * return: success or not """ api_params = dict(method = "share.addComment", content = text, \ share_id = statusID.status_id, user_id = statusID.source_user_id) try: ret = self.renren_request(api_params) logger.debug("Reply to status '%s' return: %s", statusID, ret) if 'result' in ret and ret['result'] == 1: logger.info("Reply '%s' to status '%s' succeed", text, statusID) return True else: return False except Exception, e: logger.warning("Reply failed: %s", e)
def update_func(self): logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) conn.row_factory = sqlite3.Row cursor = conn.cursor() cursor.execute("SELECT * FROM pending_update") i = cursor.fetchone() if i: cursor.execute("DELETE FROM pending_update WHERE id = ?", (i['id'], )) j = { 'id': str(i['id']), 'args': str2obj(str(i['args'])), 'kwargs': str2obj(str(i['kwargs'])), 'type': str(i['type']), 'callback': str2obj(str(i['callback'])) } res = getattr(self.sp, j['type'])(*j['args'], **j['kwargs']) if j['callback']: j['callback'](self, res) conn.commit() cursor.close() except Exception, e: logger.warning("Error while updating: %s" % (str(e)))
def _parse(self, dct): if "deleted" in dct and dct["deleted"]: logger.debug("This is a deleted message %s of SinaWeiboStatusMessage", dct["id"]) self.parsed.time = "unknown" self.parsed.username = "******" self.parsed.userid = "unknown" self.parsed.text = "unknown" self.deleted = True return self.ID.id = dct["id"] self.parsed.time = utils.str2utc(dct["created_at"]) self.parsed.username = dct["user"]["name"] self.parsed.userid = dct["user"]["id"] self.parsed.reposts_count = dct["reposts_count"] self.parsed.comments_count = dct["comments_count"] if "retweeted_status" in dct: self.parsed.username_orig = "unknown" try: self.parsed.username_orig = dct["retweeted_status"]["user"]["name"] except KeyError: logger.warning("KeyError when parsing SinaWeiboStatus. May be deleted original message") self.parsed.text_orig = dct["retweeted_status"]["text"] self.parsed.text_trace = dct["text"] self.parsed.text = ( self.parsed.text_trace + " || " + "@" + self.parsed.username_orig + " : " + self.parsed.text_orig ) else: self.parsed.text_orig = dct["text"] self.parsed.text_trace = None self.parsed.text = self.parsed.text_orig
def write_timeline_to_db(self, msglist): logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) cursor = conn.cursor() what_to_write = [] for msg in msglist: try: pickled_msg = obj2str(msg) sig = unicode(msg.digest()) cursor.execute( "SELECT * FROM home_timeline WHERE digest = ?", (sig, )) if not cursor.fetchone(): what_to_write.append( (unicode(pickled_msg), sig, msg.parsed.text, msg.parsed.username, msg.parsed.userid, msg.parsed.time)) except Exception, e: logger.warning("Error while checking message: %s" % (str(e))) try: logger.info("Writing %d messages" % (len(what_to_write))) cursor.executemany( "INSERT INTO home_timeline (pickled_object, digest, text, username, userid, time) VALUES(?, ?, ?, ?, ?, ?)", what_to_write) except Exception, e: logger.warning("Error %s" % (str(e)))
def append(self, e): if isinstance(e, Message): if hasattr(e, 'deleted') and e.deleted: logger.debug("Trying to append Deleted Message type element. Ignored") else: super(MessageList, self).append(e) else: logger.debug("Trying to append non- Message type element. Ignored")
def _parse(self, dct): if 'deleted' in dct and dct['deleted']: logger.debug( "This is a deleted message %s of SinaWeiboStatusMessage", dct["id"]) self.parsed.time = "unknown" self.parsed.username = "******" self.parsed.userid = "unknown" self.parsed.text = "unknown" self.deleted = True return self.ID.id = dct["id"] self.parsed.time = utils.str2utc(dct["created_at"]) self.parsed.username = dct['user']['name'] self.parsed.userid = dct['user']['id'] self.parsed.reposts_count = dct['reposts_count'] self.parsed.comments_count = dct['comments_count'] if 'pic_urls' in dct: for pic in dct['pic_urls']: self.parsed.attachments.append({ 'type': 'picture', 'format': ['link'], 'data': pic['thumbnail_pic'].replace('/thumbnail/', '/woriginal/') }) if 'retweeted_status' in dct: self.parsed.username_orig = "unknown" if 'pic_urls' in dct['retweeted_status']: for pic in dct['retweeted_status']['pic_urls']: self.parsed.attachments.append({ 'type': 'picture', 'format': ['link'], 'data': pic['thumbnail_pic'].replace('/thumbnail/', '/woriginal/') }) try: self.parsed.username_orig = dct['retweeted_status']['user'][ 'name'] except KeyError: logger.warning( 'KeyError when parsing SinaWeiboStatus. May be deleted original message' ) self.parsed.text_orig = dct['retweeted_status']['text'] self.parsed.text_trace = dct['text'] self.parsed.text = self.parsed.text_trace \ + " || " + "@" + self.parsed.username_orig \ + " : " + self.parsed.text_orig else: self.parsed.text_orig = dct['text'] self.parsed.text_trace = None self.parsed.text = self.parsed.text_orig
def _parse(self, dct): if 'deleted' in dct and dct['deleted']: logger.debug( "This is a deleted message %s of SinaWeiboStatusMessage", dct["id"]) self.parsed.time = "unknown" self.parsed.username = "******" self.parsed.userid = "unknown" self.parsed.text = "unknown" self.deleted = True return self.ID.id = dct["id"] self.parsed.time = utils.str2utc(dct["created_at"]) self.parsed.username = dct['user']['name'] self.parsed.userid = dct['user']['id'] self.parsed.reposts_count = dct['reposts_count'] self.parsed.comments_count = dct['comments_count'] if 'pic_urls' in dct: for pic in dct['pic_urls']: self.parsed.attachments.append({ 'type': 'picture', 'format': ['link'], 'data': pic['thumbnail_pic'].replace('/thumbnail/', '/woriginal/') }) if 'retweeted_status' in dct: self.parsed.username_orig = "unknown" if 'pic_urls' in dct['retweeted_status']: for pic in dct['retweeted_status']['pic_urls']: self.parsed.attachments.append({ 'type': 'picture', 'format': ['link'], 'data': pic['thumbnail_pic'].replace('/thumbnail/', '/woriginal/') }) try: self.parsed.username_orig = dct['retweeted_status']['user'][ 'name'] except KeyError: logger.warning( 'KeyError when parsing SinaWeiboStatus. May be deleted original message' ) self.parsed.text_orig = dct['retweeted_status']['text'] self.parsed.text_trace = dct['text'] self.parsed.text = self.parsed.text_trace \ + "//@" + self.parsed.username_orig \ + ": " + self.parsed.text_orig else: self.parsed.text_orig = dct['text'] self.parsed.text_trace = None self.parsed.text = self.parsed.text_orig
def auth(self): if self.get_saved_token(): return logger.info("Try to authenticate '%s' using OAuth2", self.jsonconf.channel_name) self.auth_first() self.auth_second() self.save_token() logger.debug("Authorized! access token is " + str(self.token)) logger.info("Channel '%s' is authorized", self.jsonconf.channel_name)
def _parse(self, dct): if 'deleted' in dct and dct['deleted']: logger.debug("This is a deleted message %s of SinaWeiboStatusMessage", dct["id"]) self.parsed.time = "unknown" self.parsed.username = "******" self.parsed.userid = "unknown" self.parsed.text = "unknown" self.deleted = True return self.ID.id = dct["id"] self.parsed.time = utils.str2utc(dct["created_at"]) self.parsed.username = dct['user']['name'] self.parsed.userid = dct['user']['id'] self.parsed.reposts_count = dct['reposts_count'] self.parsed.comments_count = dct['comments_count'] # accordian to http://open.weibo.com/qa/index.php?qa=448&qa_1=v2-%E5%B7%B2%E6%94%B6%E8%97%8F%E5%BE%AE%E5%8D%9A-%E6%8E%A5%E5%8F%A3statuses-friends-timeline%E8%BF%94%E5%9B%9E%E5%AD%97%E6%AE%B5-favorited-%E4%B8%BAfalse # Currently we have no way to tell whether # a weibo message is favorited Although there's a # specious property self.parsed.liked = False if 'pic_urls' in dct: for pic in dct['pic_urls']: self.parsed.attachments.append( { 'type': 'picture', 'format': ['link'], 'data': pic['thumbnail_pic'].replace('/thumbnail/', '/woriginal/') }) if 'retweeted_status' in dct: self.parsed.username_orig = "unknown" if 'pic_urls' in dct['retweeted_status']: for pic in dct['retweeted_status']['pic_urls']: self.parsed.attachments.append( { 'type': 'picture', 'format': ['link'], 'data': pic['thumbnail_pic'].replace('/thumbnail/', '/woriginal/') }) try: self.parsed.username_orig = dct['retweeted_status']['user']['name'] except KeyError: logger.warning('KeyError when parsing SinaWeiboStatus. May be deleted original message') self.parsed.text_orig = dct['retweeted_status']['text'] self.parsed.text_trace = dct['text'] self.parsed.text = self.parsed.text_trace \ + "//@" + self.parsed.username_orig \ + ": " + self.parsed.text_orig else: self.parsed.text_orig = dct['text'] self.parsed.text_trace = None self.parsed.text = self.parsed.text_orig
def _oauth2_second(self): ''' The second stage of oauth. Fetch authenticated code. ''' self.__init_oauth2_client() url = self.fetch_code() if url == "(null)" : raise snserror.auth self.token = self.parseCode(url) self.token.update(self.auth_client.request_access_token(self.token.code)) logger.debug("Authorized! access token is " + str(self.token)) logger.info("Channel '%s' is authorized", self.jsonconf.channel_name)
def home_timeline(self, count=20): ret = snstype.MessageList() logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) c = conn.cursor() c.execute("SELECT pickled_object FROM home_timeline ORDER BY time DESC LIMIT 0, %d" % (count,)) p = c.fetchall() logger.info("%d messages read from database" % (len(p))) for i in p: ret.append(str2obj(str(i[0]))) except Exception, e: logger.warning("Error while reading database: %s" % (str(e)))
def _oauth2_second(self): ''' The second stage of oauth. Fetch authenticated code. ''' self.__init_oauth2_client() url = self.fetch_code() if url == "(null)": raise snserror.auth self.token = self.parseCode(url) self.token.update( self.auth_client.request_access_token(self.token.code)) logger.debug("Authorized! access token is " + str(self.token)) logger.info("Channel '%s' is authorized", self.jsonconf.channel_name)
def _short_url_weibo(self, url): try: results = self.weibo_request("short_url/shorten", "GET", {"url_long": url}) logger.debug("URL shortening response: %s", results) u = results["urls"][0] return u["url_short"] # Even for usable URL, it returns False? # if u['result'] == 'true': # return u["url_short"] # else: # logger.warning("Response short URL is not usable ('%s'). Fallback to original URL", u["url_short"]) # return url except Exception as e: logger.warning("Catch exception when shortening URL on SinaWeibo: '%s'", e) return url
def home_timeline(self, count=20): ret = snstype.MessageList() logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) c = conn.cursor() c.execute( "SELECT pickled_object FROM home_timeline ORDER BY time DESC LIMIT 0, %d" % (count, )) p = c.fetchall() logger.info("%d messages read from database" % (len(p))) for i in p: ret.append(str2obj(str(i[0]))) except Exception, e: logger.warning("Error while reading database: %s" % (str(e)))
def auth(self): ''' docstring placeholder ''' if self.get_saved_token(): return logger.info("Try to authenticate '%s' using OAuth2", self.jsonconf.channel_name) self.auth_first() self.auth_second() if not self.token: return False self.save_token() logger.debug("Authorized! access token is " + str(self.token)) logger.info("Channel '%s' is authorized", self.jsonconf.channel_name)
def add_channel(self, jsonconf): logger.debug(json.dumps(jsonconf)) cname = jsonconf["channel_name"] if cname in self: logger.warning("Duplicate channel_name '%s'. Nothing happens to it. ", cname) return False try: p = getattr(platform, jsonconf["platform"]) self[cname] = p(jsonconf) self.__method_routing(cname, SNSPocket.__default_mapping) except AttributeError: logger.warning("No such platform '%s'. Nothing happens to it. ", jsonconf["platform"]) return True
def _short_url_weibo(self, url): try: results = self.weibo_request('short_url/shorten', 'GET', {'url_long': url}) logger.debug("URL shortening response: %s", results) u = results["urls"][0] return u["url_short"] # Even for usable URL, it returns False? #if u['result'] == 'true': # return u["url_short"] #else: # logger.warning("Response short URL is not usable ('%s'). Fallback to original URL", u["url_short"]) # return url except Exception as e: logger.warning( "Catch exception when shortening URL on SinaWeibo: '%s'", e) return url
def _parse(self, dct): #print dct #logger.debug("%s", dct) if 'deleted' in dct and dct['deleted']: logger.debug("This is a deleted message %s of SinaWeiboStatusMessage", dct["id"]) self.parsed.time = "unknown" self.parsed.username = "******" self.parsed.userid = "unknown" self.parsed.text = "unknown" self.deleted = True return #return snstype.DeletedMessage(dct) self.ID.id = dct["id"] self.parsed.time = utils.str2utc(dct["created_at"]) self.parsed.username = dct['user']['name'] self.parsed.userid = dct['user']['id'] #if 'user' in dct: # self.parsed.username = dct['user']['name'] # self.parsed.userid = dct['user']['id'] # logger.warning("Parsed one message with unknown 'user' for SinaWeiboStatusMessage") #else: # self.parsed.username = "******" # self.parsed.userid = "unknown" self.parsed.reposts_count = dct['reposts_count'] self.parsed.comments_count = dct['comments_count'] if 'retweeted_status' in dct: self.parsed.username_orig = "unknown" try: self.parsed.username_orig = dct['retweeted_status']['user']['name'] except KeyError: logger.warning('KeyError when parsing SinaWeiboStatus. May be deleted original message') self.parsed.text_orig = dct['retweeted_status']['text'] self.parsed.text_trace = dct['text'] self.parsed.text = self.parsed.text_trace \ + " || " + "@" + self.parsed.username_orig \ + " : " + self.parsed.text_orig else: self.parsed.text_orig = dct['text'] self.parsed.text_trace = None self.parsed.text = self.parsed.text_orig
def add_channel(self, jsonconf): logger.debug(json.dumps(jsonconf)) cname = jsonconf['channel_name'] if cname in self: logger.warning( "Duplicate channel_name '%s'. Nothing happens to it. ", cname) return False try: p = getattr(platform, jsonconf['platform']) self[cname] = p(jsonconf) self.__method_routing(cname, SNSPocket.__default_mapping) except AttributeError: logger.warning("No such platform '%s'. Nothing happens to it. ", jsonconf['platform']) return True
def _oauth2_second(self): ''' The second stage of oauth. Fetch authenticated code. ''' try: self.__init_oauth2_client() url = self.fetch_code() logger.debug("get url: %s", url) if url == "(null)" : raise snserror.auth self.token = self.parseCode(url) self.token.update(self.auth_client.request_access_token(self.token.code)) logger.debug("Authorized! access token is " + str(self.token)) logger.info("Channel '%s' is authorized", self.jsonconf.channel_name) except Exception, e: logger.warning("Auth second fail. Catch exception: %s", e) self.token = None
def home_timeline(self, count=20): ''' Get blog timeline :param count: Number of blogs ''' api_params = {'method': 'feed.get', 'type': '20,22', 'page': 1, 'count': count} try: jsonlist = self.renren_request(api_params) logger.debug("Get %d elements in response", len(jsonlist)) except RenrenAPIError, e: logger.warning("RenrenAPIError, %s", e) return snstype.MessageList()
def auth(self): ''' docstring placeholder ''' try: if self.get_saved_token(): return logger.info("Try to authenticate '%s' using OAuth2", self.jsonconf.channel_name) self.auth_first() self.auth_second() if not self.token: return False self.save_token() logger.debug("Authorized! access token is " + str(self.token)) logger.info("Channel '%s' is authorized", self.jsonconf.channel_name) except Exception, e: logger.warning("Auth second fail. Catch exception: %s", e)
def fetch_code(self): if self.auth_info.cmd_fetch_code == "(console_input)": utils.console_output( "Please input the whole url from Broswer's address bar:") return self.console_input().strip() elif self.auth_info.cmd_fetch_code == "(local_webserver)": try: self.httpd.handle_request() if 'code' in self.httpd.query_params: code = self.httpd.query_params['code'] logger.info("Get code from local server: %s", code) return "http://localhost/?%s" % urllib.urlencode( self.httpd.query_params) else: #TODO: # There is a non repeatable bug here. # When we have multiple platforms to authorize, # successive platforms may fail in this branch. # That means there is other HTTP request to the local HTTP server # before the call_back URL. # # Solution: # * Configure different port for different channels. # This is solved at upper layer. # * Support random port by default. raise snserror.auth.fetchcode finally: del self.httpd else: cmd = "%s %s" % (self.auth_info.cmd_fetch_code, self.__last_request_time) logger.debug("fetch_code command is: %s", cmd) ret = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout.readline().rstrip() tries = 1 while ret == "(null)": tries += 1 if tries > self.__fetch_code_max_try: break time.sleep(self.__fetch_code_timeout) ret = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout.read().rstrip() return ret
def _update(self, type, args, kwargs): logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) cursor = conn.cursor() callback = None if 'callback' in kwargs: callback = kwargs['callback'] del kwargs['callback'] cursor.execute( "INSERT INTO pending_update (type, callback, args, kwargs) VALUES (?, ?, ?, ?)", (type, obj2str(callback), obj2str(args), obj2str(kwargs))) conn.commit() cursor.close() return True except Exception, e: logger.warning("Error while saving pending_update: %s" % (str(e))) return False
def _replace_with_short_url(self, text): import re # TODO: # 1) This regex needs upgrade. # Is it better to match only http(s):// prefix? # 2) A better place to locate the pattern is the upper level dir, # e.g. snstype.py. URL matching pattern is universal for all # platforms. Placing it at a common area and making the pattern # testable is favourable. p = re.compile("http(s)?://[^\s]*") if isinstance(text, unicode): text = text.encode('utf-8') lst = p.findall(text) result = text for c in lst: ex_c = self._expand_url(c) surl = self._short_url_weibo(ex_c).encode('utf-8') logger.debug("url='%s', short_url='%s'", c, surl) result = result.replace(c, surl) return result.decode('utf-8')
def _replace_with_short_url(self, text): import re #TODO: # 1) This regex needs upgrade. # Is it better to match only http(s):// prefix? # 2) A better place to locate the pattern is the upper level dir, # e.g. snstype.py. URL matching pattern is universal for all # platforms. Placing it at a common area and making the pattern # testable is favourable. p = re.compile("[a-zA-z]+://[^\s]*") if isinstance(text, unicode): text = text.encode('utf-8') lst = p.findall(text) result = text for c in lst: ex_c = self._expand_url(c) surl = self._short_url_weibo(ex_c).encode('utf-8') logger.debug("url='%s', short_url='%s'", c, surl) result = result.replace(c, surl) return result.decode('utf-8')
def home_timeline(self, count=20): ''' Get blog timeline :param count: Number of blogs ''' api_params = { 'method': 'feed.get', 'type': '20,22', 'page': 1, 'count': count } try: jsonlist = self.renren_request(api_params) logger.debug("Get %d elements in response", len(jsonlist)) except RenrenAPIError, e: logger.warning("RenrenAPIError, %s", e) return snstype.MessageList()
def _fetch_code_local_username_password(self): try: login_username = self.auth_info.login_username login_password = self.auth_info.login_password app_key = self.jsonconf.app_key app_secret = self.jsonconf.app_secret callback_url = self.auth_info.callback_url referer_url = self._last_requested_url postdata = {"client_id": app_key, "redirect_uri": callback_url, "userId": login_username, "passwd": login_password, "isLoginSina": "0", "action": "submit", "response_type": "code", } headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0", "Host": "api.weibo.com", "Referer": referer_url } #TODO: # Unify all the urllib, urllib2 invocation to snsbase import urllib2 import urllib auth_url = "https://api.weibo.com/oauth2/authorize" #auth_url = self.auth_info.auth_url req = urllib2.Request(url = auth_url, data = urllib.urlencode(postdata), headers = headers ) resp = urllib2.urlopen(req) resp_url = resp.geturl() logger.debug("response URL from local post: %s", resp_url) return resp_url except Exception, e: logger.warning("Catch exception: %s", e)
def _fetch_code_local_username_password(self): try: login_username = self.auth_info.login_username login_password = self.auth_info.login_password app_key = self.jsonconf.app_key app_secret = self.jsonconf.app_secret callback_url = self.auth_info.callback_url referer_url = self._last_requested_url postdata = { "client_id": app_key, "redirect_uri": callback_url, "userId": login_username, "passwd": login_password, "isLoginSina": "0", "action": "submit", "response_type": "code", } headers = { "User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0", "Host": "api.weibo.com", "Referer": referer_url, } # TODO: # Unify all the urllib, urllib2 invocation to snsbase import urllib2 import urllib auth_url = "https://api.weibo.com/oauth2/authorize" # auth_url = self.auth_info.auth_url req = urllib2.Request(url=auth_url, data=urllib.urlencode(postdata), headers=headers) resp = urllib2.urlopen(req) resp_url = resp.geturl() logger.debug("response URL from local post: %s", resp_url) return resp_url except Exception, e: logger.warning("Catch exception: %s", e)
def auth(self): ''' docstring placeholder ''' try: if self.get_saved_token(): self.client.auth_with_token(self.token["access_token"]) return logger.info("Try to authenticate '%s' using OAuth2", self.jsonconf.channel_name) self.auth_first() self.auth_second() if not self.token: return False self.save_token() logger.debug("Authorized! access token is " + str(self.token)) logger.info("Channel '%s' is authorized", self.jsonconf.channel_name) except Exception, e: logger.warning("Auth second fail. Catch exception: %s", e)
def get_saved_token(self): try: fname = self.auth_info.save_token_file if fname == "(default)" : fname = self.jsonconf.channel_name+".token.save" if fname != "(null)" : with open(fname, "r") as fp: token = utils.JsonObject(json.load(fp)) # check expire time if self.is_expired(token): logger.debug("Saved Access token is expired, try to get one through sns.auth() :D") return False #TODO: decrypt token self.token = token else: logger.debug("This channel is configured not to save token to file") return False except IOError: logger.debug("No access token saved, try to get one through sns.auth() :D") return False logger.info("Read saved token for '%s' successfully", self.jsonconf.channel_name) print self.token return True
def get_saved_token(self): try: fname = self.auth_info.save_token_file if fname == "(default)": fname = self.jsonconf.channel_name + ".token.save" if fname != "(null)": with open(fname, "r") as fp: token = utils.JsonObject(json.load(fp)) #check expire time if self.is_expired(token): logger.debug( "Saved Access token is expired, try to get one through sns.auth() :D" ) return False #TODO: decrypt token self.token = token else: logger.debug( "This channel is configured not to save token to file") return False except IOError: logger.debug( "No access token saved, try to get one through sns.auth() :D") return False logger.info("Read saved token for '%s' successfully", self.jsonconf.channel_name) return True
def _update(self, type, args, kwargs): logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) cursor = conn.cursor() callback = None if 'callback' in kwargs: callback = kwargs['callback'] del kwargs['callback'] cursor.execute("INSERT INTO pending_update (type, callback, args, kwargs) VALUES (?, ?, ?, ?)", ( type, obj2str(callback), obj2str(args), obj2str(kwargs) )) conn.commit() cursor.close() return True except Exception, e: logger.warning("Error while saving pending_update: %s" % (str(e))) return False
def fetch_code(self): if self.auth_info.cmd_fetch_code == "(console_input)" : utils.console_output("Please input the whole url from Broswer's address bar:") return self.console_input().strip() elif self.auth_info.cmd_fetch_code == "(local_webserver)": try: self.httpd.handle_request() if 'code' in self.httpd.query_params: code = self.httpd.query_params['code'] logger.info("Get code from local server: %s", code) return "http://localhost/?%s" % urllib.urlencode(self.httpd.query_params) else: #TODO: # There is a non repeatable bug here. # When we have multiple platforms to authorize, # successive platforms may fail in this branch. # That means there is other HTTP request to the local HTTP server # before the call_back URL. # # Solution: # * Configure different port for different channels. # This is solved at upper layer. # * Support random port by default. raise snserror.auth.fetchcode finally: del self.httpd else : cmd = "%s %s" % (self.auth_info.cmd_fetch_code, self.__last_request_time) logger.debug("fetch_code command is: %s", cmd) ret = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout.readline().rstrip() tries = 1 while ret == "(null)" : tries += 1 if tries > self.__fetch_code_max_try : break time.sleep(self.__fetch_code_timeout) ret = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout.read().rstrip() return ret
def request_url(self, url): if self.auth_info.cmd_request_url == "(webbrowser)" : self.open_brower(url) elif self.auth_info.cmd_request_url == "(console_output)" : utils.console_output(url) elif self.auth_info.cmd_request_url == "(local_webserver)+(webbrowser)" : host = self.auth_info.host port = self.auth_info.port from third.server import ClientRedirectServer from third.server import ClientRedirectHandler import socket try: self.httpd = ClientRedirectServer((host, port), ClientRedirectHandler) self.open_brower(url) except socket.error: raise snserror.auth else : self.__last_request_time = self.time() cmd = "%s '%s'" % (self.auth_info.cmd_request_url, url) logger.debug("request_url command is: %s", cmd) res = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout.read().rstrip() logger.debug("request_url result is: %s", res) return
def _fetch_code_local_username_password(self): try: login_username = self.auth_info.login_username login_password = self.auth_info.login_password app_key = self.jsonconf.app_key app_secret = self.jsonconf.app_secret callback_url = self.auth_info.callback_url referer_url = self._last_requested_url postdata = { "client_id": app_key, "redirect_uri": callback_url, "userId": login_username, "passwd": login_password, "isLoginSina": "0", "action": "submit", "response_type": "code", } headers = { "User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0", "Host": "api.weibo.com", "Referer": referer_url } auth_url = "https://api.weibo.com/oauth2/authorize" #auth_url = self.auth_info.auth_url resp_url = self._http_post(auth_url, data=postdata, headers=headers, json_parse=False).url logger.debug("response URL from local post: %s", resp_url) return resp_url except Exception, e: logger.warning("Catch exception: %s", e)
def write_timeline_to_db(self, msglist): logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) cursor = conn.cursor() what_to_write = [ ] for msg in msglist: try: pickled_msg = obj2str(msg) sig = unicode(msg.digest()) cursor.execute("SELECT * FROM home_timeline WHERE digest = ?", (sig,)) if not cursor.fetchone(): what_to_write.append(( unicode(pickled_msg), sig, msg.parsed.text, msg.parsed.username, msg.parsed.userid, msg.parsed.time )) except Exception, e: logger.warning("Error while checking message: %s" % (str(e))) try: logger.info("Writing %d messages" % (len(what_to_write))) cursor.executemany("INSERT INTO home_timeline (pickled_object, digest, text, username, userid, time) VALUES(?, ?, ?, ?, ?, ?)", what_to_write) except Exception, e: logger.warning("Error %s" % (str(e)))
sys.path.append('..') from snslog import SNSLog as logger from snsbase import SNSBase, require_authed import snstype from utils import console_output import utils else: import sys from ..snslog import SNSLog as logger from ..snsbase import SNSBase, require_authed from .. import snstype from ..utils import console_output from .. import utils logger.debug("%s plugged!", __file__) # Inteface URLs. # This differs from other platforms RENREN_AUTHORIZATION_URI = "http://graph.renren.com/oauth/authorize" RENREN_ACCESS_TOKEN_URI = "http://graph.renren.com/oauth/token" RENREN_SESSION_KEY_URI = "http://graph.renren.com/renren_api/session_key" RENREN_API_SERVER = "http://api.renren.com/restserver.do" # This error is moved back to "renren.py". # It's platform specific and we do not expect other # file to raise this error. class RenrenAPIError(Exception): def __init__(self, code, message): super(RenrenAPIError, self).__init__(message) self.code = code
''' if __name__ == '__main__': import sys sys.path.append('..') from snslog import SNSLog as logger from snsbase import SNSBase, require_authed import snstype import utils else: from ..snslog import SNSLog as logger from ..snsbase import SNSBase, require_authed from .. import snstype from .. import utils logger.debug("%s plugged!", __file__) class SinaWeiboBase(SNSBase): def __init__(self, channel=None): super(SinaWeiboBase, self).__init__(channel) @staticmethod def new_channel(full=False): c = SNSBase.new_channel(full) c['app_key'] = '' c['app_secret'] = '' c['platform'] = 'SinaWeiboStatus' c['auth_info'] = { "save_token_file": "(default)",
self.__name = name def utcoffset(self, dt): return self.__offset def tzname(self, dt): return self.__name def dst(self, dt): return ZERO SNSAPI_TIMEZONE = FixedOffsetTimeZone(0, 'GMT') try: SNSAPI_TIMEZONE = tz.tzlocal() logger.debug("Get local timezone OK. Use system's tzlocal") except Exception as e: # Silently ignore it and degrades to default TZ (GMT). # Logger has not been set at the moment. # # In case other methods refer to tzlocal(), # we fix it by the default TZ configured here. # (The ``dtparser`` will refer to ``tz.tzlocal``) logger.warning("Get local timezone failed. Use default GMT") tz.tzlocal = lambda : SNSAPI_TIMEZONE def str2utc(s, tc = None): ''' :param tc: Timezone Correction (TC). A timezone suffix string. e.g. ``" +08:00"``, `` HKT``, etc.
logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) c = conn.cursor() c.execute( "SELECT pickled_object FROM home_timeline ORDER BY time DESC LIMIT 0, %d" % (count, )) p = c.fetchall() logger.info("%d messages read from database" % (len(p))) for i in p: ret.append(str2obj(str(i[0]))) except Exception, e: logger.warning("Error while reading database: %s" % (str(e))) finally: logger.debug("releasing lock") self.dblock.release() return ret def write_timeline_to_db(self, msglist): logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) cursor = conn.cursor() what_to_write = [] for msg in msglist: try: pickled_msg = obj2str(msg) sig = unicode(msg.digest()) cursor.execute(
def utcoffset(self, dt): return self.__offset def tzname(self, dt): return self.__name def dst(self, dt): return ZERO SNSAPI_TIMEZONE = FixedOffsetTimeZone(0, 'GMT') try: SNSAPI_TIMEZONE = tz.tzlocal() logger.debug("Get local timezone OK. Use system's tzlocal") except Exception as e: # Silently ignore it and degrades to default TZ (GMT). # Logger has not been set at the moment. # # In case other methods refer to tzlocal(), # we fix it by the default TZ configured here. # (The ``dtparser`` will refer to ``tz.tzlocal``) logger.warning("Get local timezone failed. Use default GMT") tz.tzlocal = lambda: SNSAPI_TIMEZONE def str2utc(s, tc=None): ''' :param tc: Timezone Correction (TC). A timezone suffix string.
import sys sys.path.append('..') from snslog import SNSLog as logger from snsbase import SNSBase, require_authed import snstype from utils import console_output import utils else: import sys from ..snslog import SNSLog as logger from ..snsbase import SNSBase, require_authed from .. import snstype from ..utils import console_output from .. import utils logger.debug("%s plugged!", __file__) # Inteface URLs. # This differs from other platforms RENREN_AUTHORIZATION_URI = "http://graph.renren.com/oauth/authorize" RENREN_ACCESS_TOKEN_URI = "http://graph.renren.com/oauth/token" RENREN_SESSION_KEY_URI = "http://graph.renren.com/renren_api/session_key" RENREN_API_SERVER = "http://api.renren.com/restserver.do" # This error is moved back to "renren.py". # It's platform specific and we do not expect other # file to raise this error. class RenrenAPIError(Exception): def __init__(self, code, message): super(RenrenAPIError, self).__init__(message)
def home_timeline(self, count=20): ret = snstype.MessageList() logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) c = conn.cursor() c.execute("SELECT pickled_object FROM home_timeline ORDER BY time DESC LIMIT 0, %d" % (count,)) p = c.fetchall() logger.info("%d messages read from database" % (len(p))) for i in p: ret.append(str2obj(str(i[0]))) except Exception, e: logger.warning("Error while reading database: %s" % (str(e))) finally: logger.debug("releasing lock") self.dblock.release() return ret def write_timeline_to_db(self, msglist): logger.debug("acquiring lock") self.dblock.acquire() try: conn = sqlite3.connect(self.sqlitefile) cursor = conn.cursor() what_to_write = [ ] for msg in msglist: try: pickled_msg = obj2str(msg) sig = unicode(msg.digest())