def post(self):
feed_id = self.get_argument('id', None)
feed_url = self.get_argument('feed_url', None)
page_url = self.get_argument('page_url', None)
if feed_id:
feed = models.Feed.get_by_id(int(feed_id))
else:
feed = models.Feed()
action = self.get_argument('action', None)
if action == 'remove':
feed.key.delete()
return self.redirect('/admin?removed=1')
if page_url:
page = urllib.urlopen(page_url).read()
soup = bs4.BeautifulSoup(page)
rss = soup.find('link', type='application/rss+xml')
feed_url = rss.get('href', None)
feed.name = self.get_argument('name', None)
feed.find_words = [
w.strip() for w in self.get_argument('find_words', '').split(',')
if w.strip()
]
feed.url = feed_url
feed.put()
self.redirect('/admin')
def initialize(self):
self.arg = {'current_user': self.current_user}
self.tg = models.TagGraph(self.r)
self.ug = models.UserGraph(self.r)
self.um = models.UserMeta(self.r)
self.wg = models.WishGraph(self.r)
self.wag = models.WishAtGraph(self.r)
self.feed = models.Feed(self.r)
self.noti = models.Notice(self.r)
def save_feed(db: Session, feed: schemas.FeedSchema):
if not get_feed_by_id_feed(db, feed.rss_id, feed.id_feed):
db_feed = models.Feed(**feed.dict())
db.add(db_feed)
db.commit()
db.refresh(db_feed)
return db_feed
else:
return False
def get(self, feed_uri):
feedset = models.Feed().all().filter('uri = ', urllib2.unquote(feed_uri)).fetch(1)
if feedset:
feed = feedset[0]
updateFeed(self, feed)
#self.redirect('/')
else:
self.response.set_status(404)
self.response.out.write("(!404) Feed not found (%s)" % urllib2.unquote(feed_uri))
def CreateFeed(url):
feed_feedparser = util.ParseFeedAtUrl(url)
feed_entity = models.Feed(key_name=url,
url=url,
title=feed_feedparser.feed.title,
link=feed_feedparser.feed.link)
UpdateFeed(feed_entity, feed_feedparser)
feed_entity.put()
return feed_entity
def update_thread():
while True:
logging.info('Checking feeds for updates')
for feed in models.Feed().select():
try:
feed.refresh()
except Exception as ex:
logging.warning(f'Exception in refresh loop ({ex})')
time.sleep(10 * 60)
def create_feed(parsed_data):
"Creates a models.Feed object from the given parsed data."
try:
# final check against the slugified title sometimes there are
# different uri's pointing to the same feed, subdomains being
# a biggie. this check might be better off elsewhere
feed = models.Feed.objects.get(slug=parsed_data['slug'])
logger.warning("feed with identical slug %r exists! cannot create." % feed.slug)
return None
except models.Feed.DoesNotExist:
# nope, definitely doesn't exist. all good :)
feed = models.Feed(**parsed_data)
feed.save()
return feed
def __init__(self):
self.feeds = []
cfg = config.config()
for section in cfg.sections():
if section == "META":
continue
d = {}
d.update(cfg.items(section))
print d
if d.has_key("crawler"):
feed = getattr(models, d['crawler'])(id=section, **d)
else:
feed = models.Feed(id=section, **d)
self.feeds.append(feed)
def get(self, user_id, feed_uri):
#user = users.get_current_user()
#if user_id == user.user_id() or users.is_current_user_admin():
#fetch the user from the user_id
userProfileSet = models.UserProfile().all().filter('id = ', user_id).fetch(1)
if userProfileSet:
user = userProfileSet[0].user
#validate that feed exists
feed_uri = urllib2.unquote(feed_uri)
if not feed_uri[:7] == 'http://':
feed_uri = 'http://' + feed_uri
feedset = models.Feed().all().filter('uri = ', feed_uri).fetch(1)
if feedset:
feed = feedset[0]
#validate that user is subscribed to feed
ufset = models.UserFeed().all().filter('user = '******'feed = ', feed).fetch(1)
if ufset:
user_feed = ufset[0]
#find if it's time to update the feed
interval = timedelta(minutes = user_feed.interval)
time_since_update = datetime.now() - user_feed.lastUpdated
if time_since_update >= interval:
if user_feed.currentItem < feed.totalItems-1:
user_feed.currentItem += 1
user_feed.lastUpdated = datetime.now()
user_feed.put()
else:
#check if we can add new items to the feed.
time_since_last_check = datetime.now() - feed.lastCheckedForUpdates
wait_period = timedelta(days = 1)
if time_since_last_check > wait_period:
updateFeed(self, feed)
#find which items should go in the feed
feed_items = feed.items.filter('num = ', user_feed.currentItem).fetch(1)
template_values = {
"feed": feed,
"feed_items": feed_items
}
self.response.headers['Content-Type'] = "text/xml; charset=utf-8"
render_template("feed.xml", template_values, self)
else:
self.response.out.write("User not registered for feed (%s)" % feed_uri)
else:
self.response.out.write("Feed not found (%s)" % feed_uri)
else:
self.response.out.write("User does not exist")
def post(self, user_id, feed_uri):
user = users.get_current_user()
if user_id == user.user_id() or users.is_current_user_admin():
feedset = models.Feed().all().filter('uri = ', urllib2.unquote(feed_uri)).fetch(1)
if feedset:
feed = feedset[0]
if self.request.get('_method') == "DELETE":
ufset = models.UserFeed().all().filter('user = '******'feed = ', feed).fetch(1)
if ufset:
ufset[0].delete()
self.redirect('/') #could sent back to user's homepage directly - save a request
else:
self.response.out.write("User not registered for feed")
else:
self.response.out.write("Malformed Request")
else:
self.response.out.write("Feed not found (%s)" % feed_uri)
else:
self.response.out.write("Not authorised")
def post(self, user_id, feed_uri):
user = users.get_current_user()
if user_id == user.user_id() or users.is_current_user_admin():
#validate that feed exists
feedset = models.Feed().all().filter('uri = ', urllib2.unquote(feed_uri)).fetch(1)
if feedset:
feed = feedset[0]
#validate that user is subscribed to feed
ufset = models.UserFeed().all().filter('user = '******'feed = ', feed).fetch(1)
if ufset:
user_feed = ufset[0]
interval = self.request.get('interval')
user_feed.interval = int(interval)
user_feed.put()
self.refdirect('/')
else:
self.response.out.write("User not registered for feed")
else:
self.response.out.write("Feed not found (%s)" % feed_uri)
else:
self.response.out.write("User does not exist")
def get(self, user_id, feed_uri):
user = users.get_current_user()
if user_id == user.user_id() or users.is_current_user_admin():
#validate that feed exists
feedset = models.Feed().all().filter('uri = ', urllib2.unquote(feed_uri)).fetch(1)
if feedset:
feed = feedset[0]
#validate that user is subscribed to feed
ufset = models.UserFeed().all().filter('user = '******'feed = ', feed).fetch(1)
if ufset:
user_feed = ufset[0]
template_values = {"user_feed": user_feed}
render_template('edit.html', template_values, self)
else:
self.response.set_status(404)
self.response.out.write("User not registered for feed")
else:
self.response.set_status(404)
self.response.out.write("(404) Feed not found (%s)" % feed_uri)
else:
self.response.set_status(404)
self.response.out.write("User does not exist")
def post(self, usernick):
user = users.get_current_user()
if (user.nickname() == usernick) or users.is_current_user_admin():
source = self.request.get('source')
interval = self.request.get('interval')
#check if feed has '?' in it. raise error if yes. (right?)
#check if feed already exists
feedset = models.Feed().all().filter('uri = ', source).fetch(1)
if not feedset:
#max 1000 items, no questions asked.
fetchFeedURI = ("http://www.google.com/reader/public/atom/feed/%s?n=1000&client=feedpacer" % source)
try:
result = urllib2.urlopen(fetchFeedURI)
except urllib2.HTTPError:
result = None
#eventually push an error.
self.redirect('/')
else:
res = result.read()
feedtree = et.XML(res)
#self.response.out.write(feedtree.attrib)
feed = models.Feed()
link = feedtree.find('{http://www.w3.org/2005/Atom}link')
#handle -feed not found- error.
if link.attrib['rel']=='alternate': feed.alt = link.attrib['href']
feed.uri = source
feed.fetched = datetime.now()
feed.title = feedtree.find('{http://www.w3.org/2005/Atom}title').text
feed.updated = datetime.strptime(feedtree.find('{http://www.w3.org/2005/Atom}updated').text,
"%Y-%m-%dT%H:%M:%SZ")
atom_id = feedtree.find('{http://www.w3.org/2005/Atom}id').text
feedlinks = feedtree.findall('{http://www.w3.org/2005/Atom}link')
for link in feedlinks:
if link.attrib['rel'] == 'self':
feed.rel_self = link.attrib['href']
elif link.attrib['rel'] == 'alternate':
feed.rel_alt = link.attrib['href']
subt = feedtree.find('{http://www.w3.org/2005/Atom}subtitle')
feed.subtitle = subt.text if subt is not None else None
feed.atom_id = 'tag:feedpacer.appspot.com' + atom_id[14:20] + atom_id[27:]
feed.idx = feedtree.attrib['{urn:atom-extension:indexing}index']
feed.dir = feedtree.attrib['{http://www.google.com/schemas/reader/atom/}dir']
entries = feedtree.findall('{http://www.w3.org/2005/Atom}entry')
feed.latestItemId = entries[0].find('{http://www.w3.org/2005/Atom}id').text
feed.checkedForUpdates = datetime.now()
feed.totalItems = len(entries)
feed.put()
ns = u'{http://www.w3.org/2005/Atom}'
nsl = len(ns)
entries.reverse()
for i,item in enumerate(entries):
feedItem = models.FeedItem()
feedItem.feed = feed
feedItem.num = i
for elem in item.getiterator():
if elem.tag.startswith(ns):
elem.tag = elem.tag[nsl:]
feedItem.whole = "<entry" + et.tostring(item)[100:]
feedItem.put()
'''atom_id = item.find('id').text
atom_id = 'tag:feedpacer.appspot.com' + atom_id[14:20] + atom_id[27:]
feedItem.atom_id = atom_id
feedItem.title = item.find('{http://www.w3.org/2005/Atom}title').text
feedItem.uri = item.find('{http://www.w3.org/2005/Atom}link').attrib['href']
feedItem.published = datetime.strptime(item.find('{http://www.w3.org/2005/Atom}published').text,
"%Y-%m-%dT%H:%M:%SZ")
feedItem.updated = datetime.strptime(item.find('{http://www.w3.org/2005/Atom}updated').text,
"%Y-%m-%dT%H:%M:%SZ")
it_sum = item.find('{http://www.w3.org/2005/Atom}summary')
it_cont = item.find('{http://www.w3.org/2005/Atom}content')
if it_sum is not None:
feedItem.summary = it_sum.text
elif it_cont is not None:
feedItem.summary = it_cont.text
else:
feedItem.summary = ""
#self.response.out.write((i, it_sum, it_cont.text))
#self.response.out.write(feedItem.summary)'''
else:
feed = feedset[0]
#check if user is already registered for this feed
ufset = models.UserFeed().all().filter('user = '******'feed = ', feed).count(1)
if (not feedset) or (not ufset):
uf = models.UserFeed()
uf.user = user
uf.feed = feed
uf.interval = int(interval)
uf.currentItem = 0
uf.lastUpdated = datetime.now()
uf.put()
self.redirect('/')
else:
self.redirect('/')
#eventually push an error 'already subscribed'.
else:
self.redirect('/')