def get(self):
self.response.headers['Content-Type'] = 'application/atom+xml'
# New style feed with user-provided app (consumer) key and secret
if (not self.request.get('consumer_key') and
not self.request.get('consumer_secret')):
# Welcome back message for old feeds
self.response.out.write("""\
<?xml version="1.0" encoding="UTF-8"?>
<feed xml:lang="en-US" xmlns="http://www.w3.org/2005/Atom">
<generator uri="https://twitter-atom.appspot.com/" version="0.1">twitter-atom</generator>
<id>https://twitter-atom.appspot.com/</id>
<title>Twitter Atom feeds is back!</title>
<updated>2013-07-08T20:00:00</updated>
<entry>
<id>tag:twitter-atom.appspot.com,2013:2</id>
<title>Twitter Atom feeds is back!</title>
<content type="xhtml">
<div xmlns="http://www.w3.org/1999/xhtml">
<p style="color: red; font-style: italic;"><b>Twitter Atom feeds is back! I'm experimenting with a new design that Twitter will (hopefully) be ok with. You can try it out by <a href="http://twitter-atom.appspot.com/">generating a new feed here</a>. Feel free to <a href="http://twitter.com/snarfed_org">ping me</a> if you have any questions. Welcome back!</b></p>
</div>
</content>
<published>2013-07-08T20:00:00</published>
</entry>
</feed>
""")
return
tw = twitter.Twitter(util.get_required_param(self, 'access_token_key'),
util.get_required_param(self, 'access_token_secret'))
list_str = self.request.get('list')
if list_str:
if list_str == 'tonysss13/financial':
raise exc.HTTPTooManyRequests("Please reduce your feed reader's polling rate.")
# this pattern is duplicated in index.html.
# also note that list names allow more characters that usernames, but the
# allowed characters aren't explicitly documented. :/ details:
# https://groups.google.com/d/topic/twitter-development-talk/lULdIVR3B9s/discussion
match = re.match(r'@?([A-Za-z0-9_]+)/([A-Za-z0-9_-]+)', list_str)
if not match:
self.abort(400, 'List must be of the form username/list (got %r)' % list_str)
user_id, group_id = match.groups()
actor = tw.get_actor(user_id)
activities = tw.get_activities(user_id=user_id, group_id=group_id, count=50)
else:
actor = tw.get_actor()
activities = tw.get_activities(count=50)
title = 'twitter-atom feed for %s' % (list_str or actor.get('username', ''))
try:
self.response.out.write(atom.activities_to_atom(
activities, actor, title=title, host_url=self.request.host_url + '/',
request_url=self.request.path_url, xml_base='https://twitter.com/'))
except DeadlineExceededError:
logging.warning('Hit 60s overall request deadline, returning 503.', exc_info=True)
raise exc.HTTPServiceUnavailable()
def get(self):
"""Handles an API GET.
Request path is of the form /site/user_id/group_id/app_id/activity_id ,
where each element except site is an optional string object id.
"""
# parse path
args = urllib.unquote(self.request.path).strip('/').split('/')
if not args or len(args) > MAX_PATH_LEN:
raise exc.HTTPNotFound('Expected 1-%d path elements; found %d' %
(MAX_PATH_LEN, len(args)))
# make source instance
site = args.pop(0)
if site == 'twitter':
src = twitter.Twitter(
access_token_key=util.get_required_param(self, 'access_token_key'),
access_token_secret=util.get_required_param(self, 'access_token_secret'))
elif site == 'facebook':
src = facebook.Facebook(
access_token=util.get_required_param(self, 'access_token'))
elif site == 'flickr':
src = flickr.Flickr(
access_token_key=util.get_required_param(self, 'access_token_key'),
access_token_secret=util.get_required_param(self, 'access_token_secret'))
elif site == 'instagram':
src = instagram.Instagram(
access_token=util.get_required_param(self, 'access_token'))
elif site == 'google+':
auth_entity = util.get_required_param(self, 'auth_entity')
src = googleplus.GooglePlus(auth_entity=ndb.Key(urlsafe=auth_entity).get())
else:
src_cls = source.sources.get(site)
if not src_cls:
raise exc.HTTPNotFound('Unknown site %r' % site)
src = src_cls(**self.request.params)
# handle default path elements
args = [None if a in defaults else a
for a, defaults in zip(args, PATH_DEFAULTS)]
user_id = args[0] if args else None
# fetch actor if necessary
actor = None
if self.request.get('format') == 'atom':
# atom needs actor
args = [None if a in defaults else a # handle default path elements
for a, defaults in zip(args, PATH_DEFAULTS)]
user_id = args[0] if args else None
actor = src.get_actor(user_id) if src else {}
# get activities and write response
response = src.get_activities_response(*args, **self.get_kwargs())
self.write_response(response, actor=actor)
def new(handler, auth_entity=None, **kwargs):
"""Creates and returns a :class:`Twitter` entity.
Args:
handler: the current :class:`webapp2.RequestHandler`
auth_entity: :class:`oauth_dropins.twitter.TwitterAuth`
kwargs: property values
"""
user = json.loads(auth_entity.user_json)
gr_source = gr_twitter.Twitter(*auth_entity.access_token())
actor = gr_source.user_to_actor(user)
return Twitter(id=user['screen_name'],
auth_entity=auth_entity.key,
url=actor.get('url'),
name=actor.get('displayName'),
picture=actor.get('image', {}).get('url'),
**kwargs)
def new(auth_entity=None, **kwargs):
"""Creates and returns a :class:`Twitter` entity.
Args:
auth_entity: :class:`oauth_dropins.twitter.TwitterAuth`
kwargs: property values
"""
assert 'username' not in kwargs
assert 'id' not in kwargs
user = json_loads(auth_entity.user_json)
gr_source = gr_twitter.Twitter(*auth_entity.access_token())
actor = gr_source.user_to_actor(user)
return Twitter(username=user['screen_name'],
auth_entity=auth_entity.key,
url=actor.get('url'),
name=actor.get('displayName'),
picture=actor.get('image', {}).get('url'),
**kwargs)
def get(self):
"""Handles an API GET.
Request path is of the form /site/user_id/group_id/app_id/activity_id ,
where each element except site is an optional string object id.
"""
# parse path
args = urllib.parse.unquote(self.request.path).strip('/').split('/')
if not args or len(args) > MAX_PATH_LEN:
raise exc.HTTPNotFound('Expected 1-%d path elements; found %d' %
(MAX_PATH_LEN, len(args)))
if len(args) > 1 and args[1] == 'nederland20':
return self.abort(
401,
'To protect our users from spam and other malicious activity, this account is temporarily locked. Please log in to https://twitter.com to unlock your account.'
)
# make source instance
site = args.pop(0)
if site == 'twitter':
src = twitter.Twitter(access_token_key=util.get_required_param(
self, 'access_token_key'),
access_token_secret=util.get_required_param(
self, 'access_token_secret'))
elif site == 'facebook':
self.abort(
400,
'Sorry, Facebook is no longer available in the REST API. Try the library instead!'
)
elif site == 'flickr':
src = flickr.Flickr(access_token_key=util.get_required_param(
self, 'access_token_key'),
access_token_secret=util.get_required_param(
self, 'access_token_secret'))
elif site == 'github':
src = github.GitHub(
access_token=util.get_required_param(self, 'access_token'))
elif site == 'instagram':
if self.request.get('interactive').lower() == 'true':
src = instagram.Instagram(scrape=True)
else:
self.abort(
400,
'Sorry, Instagram is not currently available in the REST API. Try https://instagram-atom.appspot.com/ instead!'
)
elif site == 'mastodon':
src = mastodon.Mastodon(
instance=util.get_required_param(self, 'instance'),
access_token=util.get_required_param(self, 'access_token'),
user_id=util.get_required_param(self, 'user_id'))
elif site == 'meetup':
src = meetup.Meetup(access_token_key=util.get_required_param(
self, 'access_token_key'),
access_token_secret=util.get_required_param(
self, 'access_token_secret'))
elif site == 'pixelfed':
src = pixelfed.Pixelfed(
instance=util.get_required_param(self, 'instance'),
access_token=util.get_required_param(self, 'access_token'),
user_id=util.get_required_param(self, 'user_id'))
elif site == 'reddit':
src = reddit.Reddit(refresh_token=util.get_required_param(
self, 'refresh_token'
)) # the refresh_roken should be returned but is not appearing
else:
src_cls = source.sources.get(site)
if not src_cls:
raise exc.HTTPNotFound('Unknown site %r' % site)
src = src_cls(**self.request.params)
# decode tag URI ids
for i, arg in enumerate(args):
parsed = util.parse_tag_uri(arg)
if parsed:
domain, id = parsed
if domain != src.DOMAIN:
raise exc.HTTPBadRequest(
'Expected domain %s in tag URI %s, found %s' %
(src.DOMAIN, arg, domain))
args[i] = id
# handle default path elements
args = [
None if a in defaults else a
for a, defaults in zip(args, PATH_DEFAULTS)
]
user_id = args[0] if args else None
# get activities (etc)
try:
if len(args) >= 2 and args[1] == '@blocks':
try:
response = {'items': src.get_blocklist()}
except source.RateLimited as e:
if not e.partial:
self.abort(429, str(e))
response = {'items': e.partial}
else:
response = src.get_activities_response(*args,
**self.get_kwargs())
except (NotImplementedError, ValueError) as e:
self.abort(400, str(e))
# other exceptions are handled by webutil.handlers.handle_exception(),
# which uses interpret_http_exception(), etc.
# fetch actor if necessary
actor = response.get('actor')
if not actor and self.request.get('format') == 'atom':
# atom needs actor
actor = src.get_actor(user_id) if src else {}
self.write_response(response, actor=actor, url=src.BASE_URL)
def get(self):
"""Handles an API GET.
Request path is of the form /site/user_id/group_id/app_id/activity_id ,
where each element except site is an optional string object id.
"""
# parse path
args = urllib.unquote(self.request.path).strip('/').split('/')
if not args or len(args) > MAX_PATH_LEN:
raise exc.HTTPNotFound('Expected 1-%d path elements; found %d' %
(MAX_PATH_LEN, len(args)))
# make source instance
site = args.pop(0)
if site == 'twitter':
src = twitter.Twitter(access_token_key=util.get_required_param(
self, 'access_token_key'),
access_token_secret=util.get_required_param(
self, 'access_token_secret'))
elif site == 'facebook':
src = facebook.Facebook(
access_token=util.get_required_param(self, 'access_token'))
elif site == 'flickr':
src = flickr.Flickr(access_token_key=util.get_required_param(
self, 'access_token_key'),
access_token_secret=util.get_required_param(
self, 'access_token_secret'))
elif site == 'instagram':
src = instagram.Instagram(scrape=True)
elif site == 'google+':
auth_entity = util.get_required_param(self, 'auth_entity')
src = googleplus.GooglePlus(auth_entity=ndb.Key(
urlsafe=auth_entity).get())
else:
src_cls = source.sources.get(site)
if not src_cls:
raise exc.HTTPNotFound('Unknown site %r' % site)
src = src_cls(**self.request.params)
# decode tag URI ids
for i, arg in enumerate(args):
parsed = util.parse_tag_uri(arg)
if parsed:
domain, id = parsed
if domain != src.DOMAIN:
raise exc.HTTPBadRequest(
'Expected domain %s in tag URI %s, found %s' %
(src.DOMAIN, arg, domain))
args[i] = id
# check if request is cached
cache = self.request.get('cache', '').lower() != 'false'
if cache:
cache_key = 'R %s' % self.request.path
cached = memcache.get(cache_key)
if cached:
logging.info('Serving cached response %r', cache_key)
self.write_response(cached['response'],
actor=cached['actor'],
url=src.BASE_URL)
return
# handle default path elements
args = [
None if a in defaults else a
for a, defaults in zip(args, PATH_DEFAULTS)
]
user_id = args[0] if args else None
# get activities (etc)
try:
if len(args) >= 2 and args[1] == '@blocks':
response = {'items': src.get_blocklist()}
else:
response = src.get_activities_response(*args,
**self.get_kwargs(src))
except (NotImplementedError, ValueError) as e:
self.abort(400, str(e))
# other exceptions are handled by webutil.handlers.handle_exception(),
# which uses interpret_http_exception(), etc.
# fetch actor if necessary
actor = response.get('actor')
if not actor and self.request.get('format') == 'atom':
# atom needs actor
args = [
None if a in defaults else a # handle default path elements
for a, defaults in zip(args, PATH_DEFAULTS)
]
user_id = args[0] if args else None
actor = src.get_actor(user_id) if src else {}
self.write_response(response, actor=actor, url=src.BASE_URL)
# cache response
if cache:
logging.info('Caching response in %r', cache_key)
memcache.set(cache_key, {
'response': response,
'actor': actor
}, src.RESPONSE_CACHE_TIME)
def test_like_scraping(self):
tw = twitter.Twitter(TOKEN_KEY, TOKEN_SECRET,
scrape_headers=TWITTER_SCRAPE_HEADERS)
activities = tw.get_activities(activity_id=TWEET_ID, fetch_likes=True)
likes = [t for t in activities[0]['object']['tags'] if t.get('verb') == 'like']
self.assertGreater(len(likes), 0)
if __name__ == '__main__':
# set up our timezone objects, so we can set local times on all posts
utc = tz.gettz('UTC')
est = tz.gettz('America/New_York')
# load tweet JSON from command line argument
posts = json.loads(open(sys.argv[1], 'r').read())
if not os.path.isdir('note'):
os.makedirs('note')
last_date = ''
counter = 0
for post in posts:
decoded = twitter.Twitter('token', 'secret', 'smerrill').tweet_to_activity(post)
obj = twitter.Twitter('token', 'secret', 'smerrill').tweet_to_object(post)
# first we need to deal with legacy tweets in the export, because they
# all have a published time of midnight UTC on the day of publication.
# Set them to my timezone while I'm at it.
date = datetime.datetime.strptime(decoded['published'], "%Y-%m-%d %H:%M:%S +0000").replace(tzinfo=utc).astimezone(est)
# we have the date, which is enough to build the directory path
path = "note/%s" % date.strftime('%Y/%m/%d')
if not os.path.isdir(path):
os.makedirs(path)
index = "%s/_index.md" % path
open(index,'w').close()
if last_date == date:
# this tweet has the same timestamp as the last one.
# we'll just increment by one minute. I don't recall ever
# having tweeted more than 60 times in a day.
def get(self):
"""Handles an API GET.
Request path is of the form /site/user_id/group_id/app_id/activity_id ,
where each element except site is an optional string object id.
"""
# parse path
args = urllib.unquote(self.request.path).strip('/').split('/')
if not args or len(args) > MAX_PATH_LEN:
raise exc.HTTPNotFound('Expected 1-%d path elements; found %d' %
(MAX_PATH_LEN, len(args)))
# make source instance
site = args.pop(0)
if site == 'twitter':
src = twitter.Twitter(
access_token_key=util.get_required_param(self, 'access_token_key'),
access_token_secret=util.get_required_param(self, 'access_token_secret'))
elif site == 'facebook':
src = facebook.Facebook(
access_token=util.get_required_param(self, 'access_token'))
elif site == 'flickr':
src = flickr.Flickr(
access_token_key=util.get_required_param(self, 'access_token_key'),
access_token_secret=util.get_required_param(self, 'access_token_secret'))
elif site == 'github':
src = github.GitHub(
access_token=util.get_required_param(self, 'access_token'))
elif site == 'instagram':
src = instagram.Instagram(scrape=True)
else:
src_cls = source.sources.get(site)
if not src_cls:
raise exc.HTTPNotFound('Unknown site %r' % site)
src = src_cls(**self.request.params)
# decode tag URI ids
for i, arg in enumerate(args):
parsed = util.parse_tag_uri(arg)
if parsed:
domain, id = parsed
if domain != src.DOMAIN:
raise exc.HTTPBadRequest('Expected domain %s in tag URI %s, found %s' %
(src.DOMAIN, arg, domain))
args[i] = id
# handle default path elements
args = [None if a in defaults else a
for a, defaults in zip(args, PATH_DEFAULTS)]
user_id = args[0] if args else None
# get activities (etc)
try:
if len(args) >= 2 and args[1] == '@blocks':
try:
response = {'items': src.get_blocklist()}
except source.RateLimited as e:
if not e.partial:
self.abort(429, str(e))
response = {'items': e.partial}
else:
response = src.get_activities_response(*args, **self.get_kwargs())
except (NotImplementedError, ValueError) as e:
self.abort(400, str(e))
# other exceptions are handled by webutil.handlers.handle_exception(),
# which uses interpret_http_exception(), etc.
# fetch actor if necessary
actor = response.get('actor')
if not actor and self.request.get('format') == 'atom':
# atom needs actor
args = [None if a in defaults else a # handle default path elements
for a, defaults in zip(args, PATH_DEFAULTS)]
actor = src.get_actor(user_id) if src else {}
self.write_response(response, actor=actor, url=src.BASE_URL)
