def get(self, api_method, expected_status=(200,), **extra_params): if not (api_method.startswith('http://') or api_method.startswith('https://')): api_method = '%s%s%s' % ( self.service_info['default_api_prefix'], api_method, self.service_info['default_api_suffix'] ) if self.get_cookie() : ## no token has been got so far, so try and get one from the browser cookie self.token = OAuthAccessToken.get_by_key_name(self.get_cookie()) if self.token is None: ## still no token, see if we can retrieve one for the screen_name if 'owner_screen_name' in extra_params : q = OAuthAccessToken.all() q.filter('specifier = ', extra_params['owner_screen_name']) self.token = q.get() fetch = urlfetch.fetch(self.get_signed_url( api_method, self.token, 'GET', **extra_params )) if fetch.status_code not in expected_status: raise ValueError( "Error calling... Got return status: %i [%r]" % (fetch.status_code, fetch.content) ) return decode_json(fetch.content)
def query(self, property, argv): """Make a query to Google property.""" hl = 'en' if argv[0][0:3] == 'hl=': hl = argv[0][3:] del argv[0] if argv == None: return query = ' '.join(argv) params = {'v': '1.0', 'hl':hl, 'q': self.utf8_str(query)} url = 'http://ajax.googleapis.com/ajax/services/search/%s?' % property params = urllib.urlencode(params) logging.info("GET %s", url + params) f = urllib.urlopen(url + params) res = decode_json(f.read()) logging.info("results: %r", res) if res['responseData'] and res['responseData']['results']: return res['responseData']['results'] s = '%s: %s' % (res['responseStatus'], res['responseDetails']) logging.error(s) return s
def get_tweets(): """Gather tweets from Twitter APIs and return a list of tuples of the type: (tweet date, tweet body) """ tweets = [] td_service = TwitterData.get_by_key_name(TWITTER_SERVICE_KEYNAME) if not td_service: logging.error('Twitter service data not found') return tweets td_consumer = TwitterData.get_by_key_name(TWITTER_CONSUMER_KEYNAME) if not td_consumer: logging.error('Twitter consumer data not found') return tweets timeline_url = "http://api.twitter.com/1/statuses/user_timeline.json" params = { "include_rts" : 1, "count" : 5 } client = oauth.TwitterClient(td_service.token, td_service.secret, callback_url='') contents = client.make_request(url=timeline_url, token=td_consumer.token, secret=td_consumer.secret, protected=True, additional_params=params).content for t in decode_json(contents): date = datetime.strptime(t['created_at'], TWITTER_TIME_FORMAT) tweets.append( { 'date': date, 'text':t['text']} ) return tweets
class url_client(object): support_data_type = ['json'] cookies = cookielib.CookieJar() def __init__(self, base_url, def_params = {}): self.base_url = base_url self.def_params = def_params def get_cookies(self): return self.cookies def __getattr__(self, name): name_parts = name.split('__') dt = name_parts[-1].lower() if dt in self.support_data_type: name_parts.pop() else: dt = None apiurl = '%s/%s'%(self.base_url, '/'.join(name_parts)) def wrap(**kw): extra = kw.get('extra_uri', []) url = apiurl if len(extra)==0 else '%s/%s'%(apiurl, '/'.join(map(str, extra))) params = kw.get('params', {}) params.update(self.def_params) return self.__call__(url = url, params = params, payload = kw.get('payload', ''), method = kw.get('method', 'GET'), headers = kw.get('headers', {}), data_type = dt) return wrap def __call__(self, url, params, payload, method, headers, data_type): try: if len(params) > 0: url += '?' + urllib.urlencode(params) if ('GET' == method.upper()): body = None else: if (None != payload): body = payload if isinstance(payload, str) else urllib.urlencode(payload) else: body = '' logging.debug("http url=%s"%(url)) request = urllib2.Request(url, body, headers) content = '' try: response = urllib2.urlopen(request, timeout=3) content = response.read() self.cookies.extract_cookies(response, request) except urllib2.HttpError, e: logging.error("http error: %d", e.code) except urllib2.URLError, e: logging.error("url error!") logging.debug("http response: %s"%content) # TODO: support xml .etc if data_type == 'json': return decode_json(content) else: return content
def post(self, api_method, http_method='POST', expected_status=(200,), **extra_params): if not (api_method.startswith('http://') or api_method.startswith('https://')): api_method = '%s%s%s' % ( self.service_info['default_api_prefix'], api_method, self.service_info['default_api_suffix'] ) if self.token is None: #cookie = self.get_cookie() #self.token = OAuthAccessToken.get_by_key_name(cookie['key']) raise TwitterOAuthError("You need Login.") fetch = urlfetch(url=api_method, payload=self.get_signed_body( api_method, self.token, http_method, **extra_params ), method=http_method) self.logging('POST', api_method, fetch.status_code, **extra_params) if fetch.status_code not in expected_status: raise ValueError( "Error calling... Got return status: %i [%r]" % (fetch.status_code, fetch.content) ) return decode_json(fetch.content)
def get(self, api_method, http_method='GET', expected_status=(200,), **extra_params): if not (api_method.startswith('http://') or api_method.startswith('https://')): api_method = '%s%s%s' % ( self.service_info['default_api_prefix'], api_method, self.service_info['default_api_suffix'] ) if self.token is None: #cookie = self.get_cookie() #self.token = OAuthAccessToken.get_by_key_name(cookie['key']) #self.token = self.get_access_token(self) #if not self.token: # raise TwitterOAuthError("You need Login.") raise TwitterOAuthError("You need Login.") fetch = urlfetch(self.get_signed_url( api_method, self.token, http_method, **extra_params )) self.logging('GET', api_method, fetch.status_code, **extra_params) if fetch.status_code not in expected_status: raise ValueError( "Error calling... Got return status: %i [%r]" % (fetch.status_code, fetch.content) ) return decode_json(fetch.content)
def post(self, api_method, http_method='POST', expected_status=(200,), **extra_params): if not (api_method.startswith('http://') or api_method.startswith('https://')): api_method = '%s%s%s' % ( self.service_info['default_api_prefix'], api_method, self.service_info['default_api_suffix'] ) if self.token is None: '''self.token = OAuthAccessToken.get_by_key_name(self.get_cookie())''' tokens = OAuthAccessToken.all().filter( 'email =', self.email) for token in tokens: self.token = token fetch = urlfetch(url=api_method, payload=self.get_signed_body( api_method, self.token, http_method, **extra_params ), method=http_method) if fetch.status_code not in expected_status: raise ValueError( "Error calling... Got return status: %i [%r]" % (fetch.status_code, fetch.content) ) return decode_json(fetch.content)
def post(self, api_method, http_method='POST', expected_status=(200, ), **extra_params): if not (api_method.startswith('http://') or api_method.startswith('https://')): api_method = '%s%s%s' % (self.service_info['default_api_prefix'], api_method, self.service_info['default_api_suffix']) if self.token is None: self.token = OAuthAccessToken.get_by_key_name(self.get_cookie()) fetch = urlfetch(url=api_method, payload=self.get_signed_body(api_method, self.token, http_method, **extra_params), method=http_method) if fetch.status_code not in expected_status: raise ValueError("Error calling... Got return status: %i [%r]" % (fetch.status_code, fetch.content)) return decode_json(fetch.content)
def query(self, property, argv): """Make a query to Google property.""" hl = 'en' if argv[0][0:3] == 'hl=': hl = argv[0][3:] del argv[0] if argv == None: return query = ' '.join(argv) params = {'v': '1.0', 'hl': hl, 'q': self.utf8_str(query)} url = 'http://ajax.googleapis.com/ajax/services/search/%s?' % property params = urllib.urlencode(params) logging.info("GET %s", url + params) f = urllib.urlopen(url + params) res = decode_json(f.read()) logging.info("results: %r", res) if res['responseData'] and res['responseData']['results']: return res['responseData']['results'] s = '%s: %s' % (res['responseStatus'], res['responseDetails']) logging.error(s) return s
def registerUserOld(self, phoneno, user_name, passwd): basic_auth = base64.encodestring('%s:%s' % (user_name, passwd))[:-1] request_headers = {} request_headers['Authorization'] = 'Basic %s' % basic_auth try: logging.debug("getting account credentials for user %s",user_name) resp = urlfetch.fetch('http://twitter.com/account/verify_credentials.json', headers = request_headers, deadline = 10) if resp.status_code == 200: logging.debug("user name and password are correct for %s", user_name) info = decode_json(resp.content) tuser = TwitterUser.create_by_phonenumber(phoneno, info['screen_name'], passwd) logging.debug("Stored user_name %s as against provided user name %s" % (info['screen_name'], user_name)) dstat = DailyStat.get_by_date() dstat.new_user() self.response.out.write("Congratulations !! Your twitter username is registered. Go ahead and send a twitter message by SMSing \"tweet <your twitter status\"") else: logging.warning("Submiting failed %d and response %s\n", resp.status_code,resp.content) self.response.out.write("Incorrect username/password. Note that both username and password are case sensitive. Better register online at http://www.smstweet.in") except urllib2.URLError, e: logging.error("Update: Post to twitter failed\n") self.response.out.write("Server error while posting the status. Please try again.\n")
def get(self, api_method, http_method="GET", expected_status=(200,), **extra_params): if not (api_method.startswith("http://") or api_method.startswith("https://")): api_method = "%s%s%s" % ( self.service_info["default_api_prefix"], api_method, self.service_info["default_api_suffix"], ) if self.token is None: # cookie = self.get_cookie() # self.token = OAuthAccessToken.get_by_key_name(cookie['key']) # self.token = self.get_access_token(self) # if not self.token: # raise TwitterOAuthError("You need Login.") raise TwitterOAuthError("You need Login.") fetch = urlfetch(self.get_signed_url(api_method, self.token, http_method, **extra_params)) self.logging("GET", api_method, fetch.status_code, **extra_params) if fetch.status_code not in expected_status: raise ValueError("Error calling... Got return status: %i [%r]" % (fetch.status_code, fetch.content)) return decode_json(fetch.content)
def get(self, api_method, http_method='GET', expected_status=(200,), **extra_params): if not (api_method.startswith('http://') or api_method.startswith('https://')): api_method = '%s%s%s' % ( self.service_info['default_api_prefix'], api_method, self.service_info['default_api_suffix'] ) signed_url = self.get_signed_url( api_method, self.token, http_method, **extra_params ) requestComplete = 0; error = '' while requestComplete == 0: try: fetch = urlfetch(signed_url) requestComplete = 1 error = '' except DownloadError: error = "Could not retrieve from social networks. The server could not be contacted at this time. Please try again later." if error <> '': return error if fetch.status_code not in expected_status: error = "Could not retrieve from social networks. The server could not be contacted at this time. Please try again later." return error return decode_json(fetch.content)
def query(self, property, argv): """Make a query to DuckDuckGo.""" if argv == None: return query = ' '.join(argv) params = {'o': 'json', 'q': self.utf8_str(query)} url = 'http://duckduckgo.com/?' params = urllib.urlencode(params) f = urllib.urlopen(url + params) return decode_json(f.read())
def cmd_twt(self, argv): try: url = 'http://search.twitter.com/trends.json' f = urllib.urlopen(url) res = decode_json(f.read()) response = '' for trend in res['trends']: response += '%s %s\n' % (trend['name'], trend['url']) return response except Exception, e: logging.info(e)
def cmd_twf(self, twitterid): try: url = 'http://api.twitter.com/1/statuses/friends.json?screen_name=%s' % twitterid[0] url = 'http://api.twitter.com/1/statuses/friends.json' url = self.get_signed_url(url, screen_name=twitterid[0]) f = urllib.urlopen(url) res = decode_json(f.read()) logging.debug("twu: %s" % res) if 'error' in res: return res['error'] for u in res: print u['screen_name'] except Exception, e: logging.info(e)
def cmd_T(self, argv): """Translate query""" if argv == None or len(argv) < 2: return langpair = argv[0] query = ' '.join(argv[1:]) params = {'v': '1.0', 'langpair':langpair, 'q': self.utf8_str(query)} url = 'http://ajax.googleapis.com/ajax/services/language/translate?' params = urllib.urlencode(params) f = urllib.urlopen(url + params) res = decode_json(f.read()) if res['responseData']: return res['responseData']['translatedText'] return res['responseDetails']
def cmd_twf(self, twitterid): try: url = 'http://api.twitter.com/1/statuses/friends.json?screen_name=%s' % twitterid[ 0] url = 'http://api.twitter.com/1/statuses/friends.json' url = self.get_signed_url(url, screen_name=twitterid[0]) f = urllib.urlopen(url) res = decode_json(f.read()) logging.debug("twu: %s" % res) if 'error' in res: return res['error'] for u in res: print u['screen_name'] except Exception, e: logging.info(e)
def get(self): client = OAuthClient("twitter", self) write = self.response.out.write token = self.request.get("token") if not token: self.error(403) write("Error 403 - token is not specified") return status = self.request.get("status") if not status: self.error(403) write("Error 403 - status is not specified") return access_token = client.set_token(token) if not access_token: self.error(403) write("Error 403 - token is invalid") return d = datetime.today() qtime = str(int(d.strftime("%s")) / 600) qkey = "quota:" + qtime + ":" + token quota = memcache.incr(qkey) if quota is None: memcache.add(qkey, 1, 600) elif quota > 50: self.error(503) write("Error 503 - Too many access") return try: result = client.post("/statuses/update", status=status) write("OK") except MyError, e: try: msg = decode_json(e.value) self.error(e.code) write("Error %d - %s\n" % (e.code, msg["error"])) except: self.error(e.code) write("Error %d - %s\n" % (e.code, e.value))
def get(self): client = OAuthClient('twitter', self) write = self.response.out.write token = self.request.get('token') if not token: self.error(403) write('Error 403 - token is not specified') return status = self.request.get('status') if not status: self.error(403) write('Error 403 - status is not specified') return access_token = client.set_token(token) if not access_token: self.error(403) write('Error 403 - token is invalid') return d = datetime.today() qtime = str(int(d.strftime('%s'))/600) qkey = 'quota:'+qtime+":"+token quota = memcache.incr(qkey) if quota is None: memcache.add(qkey, 1, 600) elif quota > 50: self.error(503) write("Error 503 - Too many access") return try: result = client.post('/statuses/update', status = status) write('OK') except MyError, e: try: msg = decode_json(e.value) self.error(e.code) write("Error %d - %s\n" % (e.code, msg['error'])) except: self.error(e.code) write("Error %d - %s\n" % (e.code, e.value))
def cmd_echo(self, argv): """!echo handler""" if re.match('^@[a-zA-Z]+$', argv[-1]): params = {'text':' '.join(argv[:-1]).encode('utf-8'), 'room':argv[-1][1:], 'bot':config.bot_id, 'bot_verifier':self.bot_verifier} url = 'http://lingr.com/api/room/say?' + urllib.urlencode(params) try: f = urllib2.urlopen(url) res = decode_json(f.read()) if res['status'] == 'ok': result = 'Posted' else: result = res['detail'] except Exception, e: result = str(e)
def get(self, api_method, http_method="GET", expected_status=(200,), **extra_params): if not (api_method.startswith("http://") or api_method.startswith("https://")): api_method = "%s%s%s" % ( self.service_info["default_api_prefix"], api_method, self.service_info["default_api_suffix"], ) if self.token is None: self.token = OAuthAccessToken.get_by_key_name(self.get_cookie()) fetch = urlfetch(self.get_signed_url(api_method, self.token, http_method, **extra_params)) if fetch.status_code not in expected_status: raise ValueError("Error calling... Got return status: %i [%r]" % (fetch.status_code, fetch.content)) return decode_json(fetch.content)
def cmd_tws(self, query): if len(query) == 0: return query = ' '.join(query) try: url = 'http://search.twitter.com/search.json' params = {'rpp': '3', 'q': self.utf8_str(query)} params = urllib.urlencode(params) f = urllib.urlopen(url, params) res = decode_json(f.read()) logging.info(res) response = '' for result in res['results']: response += '%s?foo.png %s %s\n' % (result['profile_image_url'], result['from_user'], result['text']) return response except Exception, e: logging.info(e)
def get(self): self.response.headers['Content-Type'] = 'text/plain' self.response.headers['charset'] = 'utf-8' self.response.set_status(200) try: json = self.request.get("json", self.request.body) logging.debug("json: %s" % json) if len(json) > 0: param = decode_json(json) for event in param['events']: self.on_message(event['message']) else: text = urllib.unquote(self.request.get("text")) self.on_message({'text':text}) except Exception, e: logging.error(e)
def get(self): self.response.headers['Content-Type'] = 'text/plain' self.response.headers['charset'] = 'utf-8' self.response.set_status(200) try: json = self.request.get("json", self.request.body) logging.debug("json: %s" % json) if len(json) > 0: param = decode_json(json) for event in param['events']: self.on_message(event['message']) else: text = urllib.unquote(self.request.get("text")) self.on_message({'text': text}) except Exception, e: logging.error(e)
def get(self, api_method, http_method="GET", expected_status=(200,), **extra_params): if not (api_method.startswith("http://") or api_method.startswith("https://")): api_method = "%s%s%s" % ( self.service_info["default_api_prefix"], api_method, self.service_info["default_api_suffix"], ) if self.token is None: raise MyError(500, "Error - token is not set") fetch = urlfetch(self.get_signed_url(api_method, self.token, http_method, **extra_params)) if fetch.status_code not in expected_status: raise MyError(fetch.status_code, fetch.content) return decode_json(fetch.content)
def post(self, api_method, http_method='POST', expected_status=(200,), **extra_params): if not (api_method.startswith('http://') or api_method.startswith('https://')): api_method = '%s%s%s' % ( self.service_info['default_api_prefix'], api_method, self.service_info['default_api_suffix'] ) if self.token is None: raise MyError(500, "token is not set") fetch = urlfetch(url=api_method, payload=self.get_signed_body( api_method, self.token, http_method, **extra_params ), method=http_method) if fetch.status_code not in expected_status: raise MyError(fetch.status_code, fetch.content) return decode_json(fetch.content)
def cmd_twu(self, twitterid): try: url = 'http://api.twitter.com/1/users/show.json' url = self.get_signed_url(url, screen_name=twitterid[0]) f = urllib.urlopen(url) res = decode_json(f.read()) logging.debug("twu: %s" % res) if 'error' in res: return res['error'] else: status = '' if res['protected']: status = "Protected" elif 'status' in res and 'text' in res['status']: status = res['status']['text'] return self.twu_format(res['profile_image_url'], res['screen_name'], res['name'], res['location'], res['url'], res['description'], status, res['friends_count'], res['followers_count'], res['statuses_count'], res['verified']) except Exception, e: logging.info(e)
def cmd_echo(self, argv): """!echo handler""" if re.match('^@[a-zA-Z]+$', argv[-1]): params = { 'text': ' '.join(argv[:-1]).encode('utf-8'), 'room': argv[-1][1:], 'bot': config.bot_id, 'bot_verifier': self.bot_verifier } url = 'http://lingr.com/api/room/say?' + urllib.urlencode(params) try: f = urllib2.urlopen(url) res = decode_json(f.read()) if res['status'] == 'ok': result = 'Posted' else: result = res['detail'] except Exception, e: result = str(e)
def cmd_tweet(self, argv): tweetid = argv[0] try: url = 'http://api.twitter.com/1/statuses/show/%s.json' % tweetid url = 'http://api.twitter.com/1/statuses/show.json' url = self.get_signed_url(url, id=tweetid, include_entities='true') f = urllib.urlopen(url) res = decode_json(f.read()) logging.debug('json: %r', res) if 'error' in res: return res['error'] media_urls = [] text = res['text'] try: for media in res['entities']['media']: text = text.replace(media['url'], media['display_url']) media_urls.append(media['media_url']) except: pass response = '' response = "%s?foo.png %s\n" % (res['user']['profile_image_url'], res['user']['screen_name']) response += "%s\n" % text response += self.relative_timestamp(res['created_at']) if res['source']: match = re.match('.*>(.+)</a>', res['source']) if match: source = match.group(1) else: source = res['source'] response += " via %s" % source if res['in_reply_to_screen_name']: response += " in reply to %s" % res['in_reply_to_screen_name'] if len(media_urls): response += "\n%s" % "\n".join(media_urls) return response except Exception, e: logging.info(e) return e
def cmd_tws(self, query): if len(query) == 0: return query = ' '.join(query) try: url = 'http://search.twitter.com/search.json' params = {'rpp': '3', 'q': self.utf8_str(query)} params = urllib.urlencode(params) f = urllib.urlopen(url, params) res = decode_json(f.read()) logging.info(res) response = '' for result in res['results']: response += '%s?foo.png %s %s\n' % ( result['profile_image_url'], result['from_user'], result['text']) return response except Exception, e: logging.info(e)
def post_update(self, access_token, status): if access_token is None: raise TwitterNotAuthorisedError("You need to authorise this Twitter account") url = 'http://api.twitter.com/1/statuses/update.json' if len(status) > CHARACTER_LIMIT: raise TwitterError("Text must be less than or equal to %d characters. " "Consider using PostUpdates." % CHARACTER_LIMIT) data = {'status': status} fetch = urlfetch(url, payload=self.get_signed_body(url, access_token, 'POST', **data), method=POST) expected_status =(200,) if fetch.status_code not in expected_status: raise ValueError( "Error calling... Got return status: %i [%r]" % (fetch.status_code, fetch.content) ) return decode_json(fetch.content)
def get(self, api_method, **extra_params): if not (api_method.startswith('http://') or api_method.startswith('https://')): api_method = '%s%s%s' % ( self.service_info['default_api_prefix'], api_method, self.service_info['default_api_suffix'] ) if self.token is None: self.token = OAuthAccessToken.get_by_key_name(self.get_cookie()) fetch = urlfetch(self.get_signed_url( api_method, self.token, **extra_params )) if fetch.status_code != 200: raise ValueError( "Error calling... Got return status: %i [%r]" % (fetch.status_code, fetch.content) ) return decode_json(fetch.content)
def post(self, api_method, http_method='POST', expected_status=(200,), **extra_params): if not (api_method.startswith('http://') or api_method.startswith('https://')): api_method = '%s%s%s' % ( self.service_info['default_api_prefix'], api_method, self.service_info['default_api_suffix'] ) body = self.get_signed_body(api_method, self.token, http_method, **extra_params) requestComplete = 0; while requestComplete == 0: try: fetch = urlfetch(url=api_method, payload=body, method=http_method) requestComplete = 1; except DownloadError: return "Could not post to social networks. The server could not be contacted at this time. Please try again later." if fetch.status_code not in expected_status: return "Could not post to social networks. The server could not be contacted at this time. Please try again later." return decode_json(fetch.content)
def cmd_twu(self, twitterid): try: url = 'http://api.twitter.com/1/users/show.json' url = self.get_signed_url(url, screen_name=twitterid[0]) f = urllib.urlopen(url) res = decode_json(f.read()) logging.debug("twu: %s" % res) if 'error' in res: return res['error'] else: status = '' if res['protected']: status = "Protected" elif 'status' in res and 'text' in res['status']: status = res['status']['text'] return self.twu_format( res['profile_image_url'], res['screen_name'], res['name'], res['location'], res['url'], res['description'], status, res['friends_count'], res['followers_count'], res['statuses_count'], res['verified']) except Exception, e: logging.info(e)
from wsgiref.handlers import CGIHandler sys.path.insert(0, join_path(dirname(__file__), "lib")) # extend sys.path from demjson import decode as decode_json from demjson import encode as encode_json from google.appengine.api.urlfetch import fetch as urlfetch, GET, POST from google.appengine.ext import db from google.appengine.ext.webapp import RequestHandler, WSGIApplication # ------------------------------------------------------------------------------ # configuration -- SET THESE TO SUIT YOUR APP!! # ------------------------------------------------------------------------------ OAUTH_APP_SETTINGS = decode_json(open("config.json").read()) CLEANUP_BATCH_SIZE = 100 EXPIRATION_WINDOW = timedelta(seconds=60 * 60 * 1) # 1 hour try: from config import OAUTH_APP_SETTINGS except: pass STATIC_OAUTH_TIMESTAMP = 12345 # a workaround for clock skew/network lag # ------------------------------------------------------------------------------ # utility functions # ------------------------------------------------------------------------------