def create_feed(user,board_id,title,link,description,actions,public_board,get_all,token):
""" Creates a feed and attaches it to a user object """
user = get_user(user)
q = Feed.query(
Feed.channel_title==title,
Feed.channel_link==link,
Feed.channel_description==description,
Feed.feed_name==title,
Feed.token==token,
Feed.actions.IN(actions),
Feed.board_id==board_id,
Feed.public_board==public_board)
feed = q.get()
if feed is None:
feed = Feed(
channel_title=title,
channel_link=link,
channel_description=description,
feed_name=title,
token=token,
actions=actions,
board_id=board_id,
public_board=public_board,
get_all=get_all,user=user.key)
feed.put()
if user.feeds:
if feed.key not in user.feeds:
user.feeds.append(feed.key)
user.put()
else:
user.feeds = [feed.key]
user.put()
return create_url(feed.key.id())
def all_feeds():
"""Post all new items for feeds for a specific interval"""
def feed_to_dict(feed):
return {
'feed_key': feed.key.urlsafe(),
'feed_url': feed.feed_url,
'etag': feed.etag,
'last_hash': feed.last_fetched_content_hash,
}
qit = Feed.query().iter()
feeds_response = []
while (yield qit.has_next_async()):
feeds_response.append(feed_to_dict(qit.next()))
poller_run_id = uuid.uuid4().hex
logger.info('Poller run %s dispatched with %d feeds', poller_run_id, len(feeds_response))
response = {
'poller_run_id': poller_run_id,
'feeds': feeds_response,
}
raise ndb.Return(jsonify(status='ok', data=response))
def try_push_resub():
"""Post all new items for feeds for a specific interval"""
if request.headers.get('X-Appengine-Cron') != 'true':
raise ndb.Return(jsonify_error(message='Not a cron call'))
unsubscribed_feeds = Feed.query(Feed.hub != None, Feed.subscribed_at_hub == False) # noqa
qit = unsubscribed_feeds.iter()
errors = 0
success = 0
count = 0
futures = []
while (yield qit.has_next_async()):
feed = qit.next()
futures.append((feed, Feed.subscribe_to_hub(feed)))
for feed, future in futures:
count += 1
try:
yield future
success += 1
except:
errors += 1
logger.exception('Failed to PuSH subscribe feed:%s' % (feed.feed_url, ))
logger.info('Tried to call hub for num_unsubscribed_feeds:%s success:%s, errors:%s', count, success, errors)
raise ndb.Return(jsonify(status='ok'))
def post_all_feeds():
"""Post all new items for feeds for a specific interval"""
if request.headers.get('X-Appengine-Cron') != 'true':
raise ndb.Return(jsonify_error(message='Not a cron call'))
feeds = Feed.query().iter()
errors = 0
success = 0
num_posted = 0
futures = []
while (yield feeds.has_next_async()):
feed = feeds.next()
futures.append((feed, Entry.publish_for_feed(feed)))
for feed, future in futures:
try:
num_posts = yield future
if num_posts is not None:
num_posted += num_posts
success += 1
except:
errors += 1
logger.exception('Failed to Publish feed:%s' % (feed.feed_url, ))
logger.info('Post Feeds success:%s errors: %s num_posted: %s', success, errors, num_posted)
raise ndb.Return(jsonify(status='ok'))
def rpc_fetch():
q = Feed.query()
results = ndb.get_multi(q.fetch(keys_only=True))
rpcs = []
for f in results:
rpc = urlfetch.create_rpc()
urlfetch.make_fetch_call(rpc, f.url)
rpcs.append(rpc)
for rpc in rpcs:
rpc.wait()
result = rpc.get_result()
d = feedparser.parse(result.content)
for e in d['entries']:
dt = parser.parse(e["published"]).replace(tzinfo=None)
dy = (datetime.datetime.utcnow() - datetime.timedelta(days=COLLECT_DAYS, seconds=COLLECT_HOURS*3600)).replace(tzinfo=None)
if dt > dy:
obj = EntryCollect.get_or_insert(e["id"])
if obj.published and obj.published >= dt:
pass
else:
logging.info("new entry : %s" % e["id"])
obj.published = dt
obj.title = e["title"]
obj.link = e["link"]
obj.summary = clean_html(e["summary"])
obj.feed = d['feed']['title']
obj.need_collect_word = True
obj.need_notice = True
obj.put()
def getFeedsCreated(self, request):
"""Return feeds created by user."""
# make sure user is authed
user = endpoints.get_current_user()
if not user:
raise endpoints.UnauthorizedException('Authorization required')
# make profile key
p_key = ndb.Key(Profile, getUserId(user))
# create ancestor query for this user
feeds = Feed.query(ancestor=p_key)
return FeedForms(items=[self._copyFeedToForm(feed) for feed in feeds])
def post(self):
"""Create a new feed and save it to the database"""
feed_url = self.request.get('feed_url')
logging.info("URL POSTed: %s" % feed_url)
if feed_url:
# Check if we already have this feed
qry = Feed.query(Feed.url == feed_url)
if qry.count() == 0:
# Retrieve the feed details and then save it to the Datastore
try:
feed = create_feed_from_url(feed_url)
# Do another check to see if we're following this feed
# The URL may have redirected
qry = Feed.query(Feed.url == feed.url)
if qry.count() > 0:
return webapp2.Response(status=409,
body="You're already following that feed")
# Set the date of last post to now so we only look for future posts
feed.last_checked = datetime.min
feed.put()
self.response.headers['Content-Type'] = 'application/json'
obj = {
'feed_name': feed.name,
'feed_url': feed.url
}
self.response.out.write(json.dumps(obj))
except Exception as e:
return webapp2.Response(status=400, body=e.message)
else:
return webapp2.Response(status=409, body="You're already following that feed")
else:
return webapp2.Response(status=400, body="Missing parameter 'feed_url'")
def get(self):
feeds_query = Feed.query(ancestor=root_key())
feeds = feeds_query.fetch()
template_values = {
'feeds': feeds,
}
# A logout URL is useful during dev
DEV = os.environ['SERVER_SOFTWARE'].startswith('Development')
if DEV:
template_values['logout_url'] = users.create_logout_url('/')
template = JINJA_ENVIRONMENT.get_template('templates/feeds.html')
self.response.write(template.render(template_values))
def delete(self):
"""Delete a feed from the database"""
feed_url = self.request.get('feed_url')
logging.info("URL DELETEed: %s" % feed_url)
if feed_url:
unquoted_url = urllib.unquote(feed_url)
unescaped_url = HTMLParser.HTMLParser().unescape(unquoted_url)
qry = Feed.query(Feed.url == unescaped_url)
if qry.count():
feed = qry.get()
feed.key.delete()
else:
return webapp2.Response(status=404, body="Feed doesn't exist")
else:
return webapp2.Response(status=400, body="Missing parameter 'feed_url'")
def all_feeds():
"""Post all new items for feeds for a specific interval"""
access_token = request.headers.get('Authorization', '').replace('Bearer ', '')
ctx = ndb.get_context()
resp = yield ctx.urlfetch('https://alpha-api.app.net/stream/0/token', headers={'Authorization': 'Bearer %s' % (access_token)})
if resp.status_code != 200:
logger.info('Token call failed: %s' % resp.content)
raise ndb.Return(jsonify_error('Not Authorized', code=401))
token = json.loads(resp.content).get('data', {})
if token.get('app', {}).get('client_id') != app.config['CLIENT_ID']:
logger.info('Client ID mismatch: %s' % resp.content)
raise ndb.Return(jsonify_error('Not Authorized', code=401))
if token.get('user'):
logger.info('Token had a user not an app token: %s' % resp.content)
raise ndb.Return(jsonify_error('Not Authorized', code=401))
feeds = Feed.query().iter()
feeds_response = []
def feed_to_dict(feed):
return {
'feed_key': feed.key.urlsafe(),
'feed_url': feed.feed_url,
'etag': feed.etag,
'last_hash': feed.last_fetched_content_hash,
}
while (yield feeds.has_next_async()):
feed = feeds.next()
feeds_response.append(feed_to_dict(feed))
logger.info('Returning all the feeds')
raise ndb.Return(jsonify(status='ok', data=feeds_response))
def get_feeds():
feeds = Feed.query().order(Feed.title)
entries = Entry.query().order(-Entry.published)
return feeds, entries
def get(self):
an_hour_ago = dt.datetime.now() - dt.timedelta(hours=1)
feeds = Feed.query(Feed.checked < an_hour_ago)
for feed in feeds:
self.update(feed.link)
def queryFeeds(self, request):
""" query all feeds added by all users"""
feeds = Feed.query()
return FeedForms(
items=[self._copyFeedToForm(feed) for feed in feeds])
