def get_salmon(self): """Returns a list of Salmon template var dicts for posts and their comments.""" resp = json.loads(util.urlfetch( API_LINKS_URL % {'id': self.key().name(), 'access_token': self.access_token})) return list(itertools.chain(*[self.post_and_comments_to_salmon_vars(post) for post in resp['data']]))
def get_salmon(self): """Returns a list of Salmon template var dicts for tweets and replies.""" # find tweets with links that include our base url. response is JSON tweets: # https://dev.twitter.com/docs/api/1/get/search resp = util.urlfetch(API_SEARCH_URL % self.key().name()) tweets = json.loads(resp)['results'] # twitter usernames of people who wrote tweets with links to our domain author_usernames = set() # maps tweet id to the link (to our domain) that it contains tweets_to_links = {} # get tweets that link to our domain salmons = [] for tweet in tweets: id = tweet.get('id') username = tweet.get('from_user') tweet_url = self.tweet_url(username, id) salmon = self.tweet_to_salmon_vars(tweet) if salmon['in_reply_to']: logging.debug('Found link %s in tweet %s', salmon['in_reply_to'], tweet_url) salmons.append(salmon) author_usernames.add(username) tweets_to_links[id] = salmon['in_reply_to'] else: logging.info("Tweet %s should have %s link but doesn't. Maybe shortened?", tweet_url, self.key().name()) # find replies to those tweets by searching for tweets that mention the # authors. for username in author_usernames: resp = util.urlfetch(API_SEARCH_URL % ('@' + username)) mentions = json.loads(resp)['results'] for mention in mentions: logging.debug('Looking at mention: %s', mention) link = tweets_to_links.get(mention.get('in_reply_to_status_id')) if link: salmon = self.tweet_to_salmon_vars(mention) salmon['in_reply_to'] = link salmons.append(salmon) logging.debug('Found reply %s', self.tweet_url(mention['from_user'], mention['id'])) return salmons
def discover_salmon_endpoint(self): """Discovers and returns the Salmon endpoint URL for this salmon. It'd be nice to use an XRD/LRDD library here, but I haven't found much. github.com/jcarbaugh/python-xrd is only for generating, not reading. pydataportability.net looks powerful but also crazy heavyweight; it requires Zope and strongly recommends running inside virtualenv. No thanks. Returns: string URL or None """ url = json.loads(self.vars)["in_reply_to"] logging.debug("Discovering salmon endpoint for %r", url) body = util.urlfetch(url) # first look in the document itself endpoint = django_salmon.discover_salmon_endpoint(body) if endpoint: logging.debug("Found in original document: %r", endpoint) return endpoint # next, look in its feed, if any # # background on feed autodiscovery: # http://blog.whatwg.org/feed-autodiscovery parsed = feedparser.parse(body) for link in parsed.feed.get("links", []): rels = link.get("rel").split() href = link.get("href") if href and ("feed" in rels or "alternate" in rels): endpoint = django_salmon.discover_salmon_endpoint(util.urlfetch(href)) if endpoint: logging.debug("Found in feed: %r", endpoint) return endpoint # next, look in /.well-known/host-meta host_meta_url = "http://%s/.well-known/host-meta" % util.domain_from_link(url) endpoint = django_salmon.discover_salmon_endpoint(util.urlfetch(host_meta_url)) if endpoint: logging.debug("Found in host-meta: %r", endpoint) return endpoint logging.debug("No salmon endpoint found!") return None
def get_posts(self, migration, scan_url=None): """Fetches a page of posts. Args: migration: Migration scan_url: string, the API URL to fetch the current page of posts. If None, starts at the beginning. Returns: (posts, next_scan_url). posts is a sequence of FacebookPosts. next_scan_url is a string, the API URL to use for the next scan, or None if there is nothing more to scan. """ # TODO: expose these as options # Publish these post types. POST_TYPES = ('link', 'checkin', 'video') # , 'photo', 'status', ... # Publish these status types. STATUS_TYPES = ('shared_story', 'added_photos', 'mobile_status_update') # 'wall_post', 'approved_friend', 'created_note', 'tagged_in_photo', ... # Don't publish posts from these applications APPLICATION_BLACKLIST = ('Likes', 'Links', 'twitterfeed') if not scan_url: scan_url = API_POSTS_URL % { 'id': self.key().name(), 'access_token': self.access_token } resp = json.loads(util.urlfetch(scan_url)) posts = [] for post in resp['data']: app = post.get('application', {}).get('name') if ((post.get('type') not in POST_TYPES and post.get('status_type') not in STATUS_TYPES) or (app and app in APPLICATION_BLACKLIST) or # posts with 'story' aren't explicit posts. they're friend approvals or # likes or photo tags or comments on other people's posts. 'story' in post): logging.info('Skipping post %s', post.get('id')) continue posts.append( FacebookPost(key_name_parts=(post['id'], migration.key().name()), json_data=json.dumps(post))) next_scan_url = resp.get('paging', {}).get('next') # XXX remove if posts and posts[-1].data()['created_time'] < '2013-09-01': next_scan_url = None # XXX return posts, next_scan_url
def get_posts(self, migration, scan_url=None): """Fetches a page of posts. Args: migration: Migration scan_url: string, the API URL to fetch the current page of posts. If None, starts at the beginning. Returns: (posts, next_scan_url). posts is a sequence of FacebookPosts. next_scan_url is a string, the API URL to use for the next scan, or None if there is nothing more to scan. """ # TODO: expose these as options # Publish these post types. POST_TYPES = ('link', 'checkin', 'video') # , 'photo', 'status', ... # Publish these status types. STATUS_TYPES = ('shared_story', 'added_photos', 'mobile_status_update') # 'wall_post', 'approved_friend', 'created_note', 'tagged_in_photo', ... # Don't publish posts from these applications APPLICATION_BLACKLIST = ('Likes', 'Links', 'twitterfeed') if not scan_url: scan_url = API_POSTS_URL % {'id': self.key().name(), 'access_token': self.access_token} resp = json.loads(util.urlfetch(scan_url)) posts = [] for post in resp['data']: app = post.get('application', {}).get('name') if ((post.get('type') not in POST_TYPES and post.get('status_type') not in STATUS_TYPES) or (app and app in APPLICATION_BLACKLIST) or # posts with 'story' aren't explicit posts. they're friend approvals or # likes or photo tags or comments on other people's posts. 'story' in post): logging.info('Skipping post %s', post.get('id')) continue posts.append(FacebookPost(key_name_parts=(post['id'], migration.key().name()), json_data=json.dumps(post))) next_scan_url = resp.get('paging', {}).get('next') # XXX remove if posts and posts[-1].data()['created_time'] < '2013-09-01': next_scan_url = None # XXX return posts, next_scan_url
def new(handler, access_token=None): """Creates and returns a Facebook instance for the logged in user. Args: handler: the current webapp2.RequestHandler """ assert access_token resp = util.urlfetch(API_USER_URL % {'id': 'me', 'access_token': access_token}) me = json.loads(resp) id = me['id'] return Facebook.get_or_insert( id, access_token=access_token, name=me.get('name'), picture='https://graph.facebook.com/%s/picture?type=small' % id, url='http://facebook.com/%s' % id)
def new(handler): """Creates and returns a Facebook for the logged in user. Args: handler: the current webapp2.RequestHandler """ access_token = handler.request.get('access_token') resp = util.urlfetch(API_USER_URL % {'id': 'me', 'access_token': access_token}) me = json.loads(resp) id = me['id'] return Facebook( key_name=id, owner=models.User.get_or_insert_current_user(handler), access_token=access_token, name=me.get('name'), picture='https://graph.facebook.com/%s/picture?type=small' % id, url='http://facebook.com/%s' % id)
def new(handler, access_token=None): """Creates and returns a Facebook instance for the logged in user. Args: handler: the current webapp2.RequestHandler """ assert access_token resp = util.urlfetch(API_USER_URL % { 'id': 'me', 'access_token': access_token }) me = json.loads(resp) id = me['id'] return Facebook.get_or_insert( id, access_token=access_token, name=me.get('name'), picture='https://graph.facebook.com/%s/picture?type=small' % id, url='http://facebook.com/%s' % id)
def envelope(self): """Signs and encloses this salmon in a Magic Signature envelope. Fetches the author's Magic Signatures public key via LRDD in order to create the signature. Returns: JSON dict following Magic Signatures spec section 3.5: http://salmon-protocol.googlecode.com/svn/trunk/draft-panzer-magicsig-01.html#anchor5 """ class UserKey(object): def __init__(self, **kwargs): vars(self).update(**kwargs) salmon_vars = json.loads(self.vars) key_url = USER_KEY_HANDLER % (salmon_vars["author_uri"], appengine_config.USER_KEY_HANDLER_SECRET) key = UserKey(**json.loads(util.urlfetch(key_url))) return XML_DOCTYPE_LINE + magicsigs.magic_envelope( ATOM_SALMON_TEMPLATE % salmon_vars, "application/atom+xml", key )