def update_feed_xml(feed_uid, feed_xml):
"""Update a feed URL and fetch the feed. Returns the number of new items"""
feed_uid = int(feed_uid)
r = requests.get(feed_xml)
f = feedparser.parse(r.content)
if not f.feed:
raise ParseError
normalize.normalize_feed(f)
with dbop.db() as db:
c = db.cursor()
clear_errors(db, c, feed_uid, f)
try:
c.execute("""update fm_feeds set feed_xml=?, feed_html=?
where feed_uid=?""",
[feed_xml, str(f.feed['link']), feed_uid])
except sqlite3.IntegrityError, e:
if 'feed_xml' in str(e):
db.rollback()
raise FeedAlreadyExists
else:
db.rollback()
raise UnknownError(str(e))
filters.load_rules(c)
num_added = process_parsed_feed(db, c, f, feed_uid)
db.commit()
return num_added
def update_item(item_uid, link, title, content):
item_uid = int(item_uid)
with dbop.db() as db:
db.execute(
"""update fm_items set item_link=?, item_title=?, item_content=?
where item_uid=?""", [link, title, content, item_uid])
db.commit()
def title_url(feed_uid):
feed_uid = int(feed_uid)
with dbop.db() as db:
c = db.execute("""select feed_title, feed_html from fm_feeds
where feed_uid=?""",
[feed_uid])
return c.fetchone()
def update_feed_private(feed_uid, private):
feed_uid = int(feed_uid)
private = int(bool(private))
with dbop.db() as db:
db.execute("update fm_feeds set feed_private=? where feed_uid=?",
[private, feed_uid])
db.commit()
def purge_reload(feed_uid):
reload(transform)
feed_uid = int(feed_uid)
if feed_uid in feed_guid_cache:
del feed_guid_cache[feed_uid]
with dbop.db() as db:
c = db.cursor()
# refresh filtering rules
filters.load_rules(c)
c.execute("delete from fm_items where item_feed_uid=? and item_rating=0",
[feed_uid])
c.execute("""delete from fm_tags
where exists (
select item_uid from fm_items
where item_uid=tag_item_uid and item_feed_uid=? and item_rating=0
)""", [feed_uid])
c.execute("""update fm_feeds set feed_modified=NULL, feed_etag=NULL
where feed_uid=?""", [feed_uid])
c.execute("""select feed_xml from fm_feeds
where feed_uid=?""", [feed_uid])
feed_xml = c.fetchone()[0]
db.commit()
r = requests.get(feed_xml)
f = feedparser.parse(r.content)
if not f.feed:
raise ParseError
normalize.normalize_feed(f)
clear_errors(db, c, feed_uid, f)
filters.load_rules(c)
num_added = process_parsed_feed(db, c, f, feed_uid)
db.commit()
def catch_up(feed_uid):
feed_uid = int(feed_uid)
with dbop.db() as db:
db.execute(
"""update fm_items set item_rating=-1
where item_feed_uid=? and item_rating=0""", [feed_uid])
db.commit()
def set_status(feed_uid, status):
feed_uid = int(feed_uid)
status = int(status)
with dbop.db() as db:
db.execute("update fm_feeds set feed_status=? where feed_uid=?",
[status, feed_uid])
db.commit()
def facebook():
with dbop.db() as db:
c = db.cursor()
app_id = param.settings.get('fb_app_id', '')
secret = param.settings.get('fb_secret', '')
if app_id and secret:
## XXX cannot assume https://
redir = 'https://' + flask.request.headers['host'] \
+ '/facebook?op=oauth_redirect'
op = flask.request.args.get('op', '')
if not op:
fb_url = 'https://graph.facebook.com/oauth/authorize?display=touch&client_id=' + app_id + '&scope=publish_pages,manage_pages&redirect_uri=' + redir
print >> param.log, 'FB_URL =', fb_url
return flask.redirect(fb_url)
elif op == 'oauth_redirect':
code = flask.request.args.get('code', '')
if code:
r = requests.get(
'https://graph.facebook.com/oauth/access_token',
params={
'client_id': app_id,
'client_secret': secret,
'code': code,
'redirect_uri': redir
})
print >> param.log, 'FACEBOOK TOKEN RESPONSE', r.text
if r.text.startswith('{'):
token = json.loads(r.text).get('access_token')
else:
token = r.text.split('access_token=', 1)[-1]
dbop.setting(db, c, fb_token=token)
return flask.redirect('/settings#facebook')
else:
return settings(status='You need to set the App ID first')
def cleanup(db=None, c=None):
"""garbage collection - see param.py
this is done only once a day between 3 and 4 AM as this is quite intensive
and could interfere with user activity
It can also be invoked by running temboz --clean
"""
if not db:
with dbop.db() as db:
c = db.cursor()
return cleanup(db, c)
# XXX need to use PATH instead
sqlite_cli = '/usr/local/bin/sqlite3'
if getattr(param, 'garbage_contents', False):
c.execute("""update fm_items set item_content=''
where item_rating < 0 and item_created < julianday('now')-?""",
[param.garbage_contents])
db.commit()
if getattr(param, 'garbage_items', False):
c.execute("""delete from fm_items where item_uid in (
select item_uid from fm_items, fm_feeds
where item_created < min(julianday('now')-?, feed_oldest-7)
and item_rating<0 and feed_uid=item_feed_uid)""", [param.garbage_items])
db.commit()
dbop.snr_mv(db, c)
c.execute("""delete from fm_tags
where not exists(
select item_uid from fm_items where item_uid=tag_item_uid
)""")
db.commit()
if dbop.fts_enabled:
c.execute("""insert into search(search) values ('rebuild')""")
c.execute('vacuum')
# we still hold the PseudoCursor lock, this is a good opportunity to backup
try:
os.mkdir('backups')
except OSError:
pass
prune_feed_guid_cache()
os.system((sqlite_cli + ' rss.db .dump | %s > backups/daily_' \
+ time.strftime('%Y-%m-%d') + '%s') % param.backup_compressor)
# rotate the log
os.rename(param.log_filename, 'backups/log_' + time.strftime('%Y-%m-%d'))
param.log.close()
param.log = open(param.log_filename, 'a', 0)
os.dup2(param.log.fileno(), 1)
os.dup2(param.log.fileno(), 2)
# delete old backups
backup_re = re.compile(
'daily_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\\.')
log_re = re.compile(
'log_[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]')
for fn in os.listdir('backups'):
if backup_re.match(fn) or log_re.match(fn):
elapsed = time.time() - os.stat('backups/' + fn).st_ctime
if elapsed > 86400 * param.daily_backups:
try:
os.remove('backups/' + fn)
except OSError:
pass
def blogroll():
cols = ('uid', 'title', 'description', 'html', 'xml', 'snr')
with dbop.db() as db:
rows = dbop.opml(db)
return (json.dumps([dict(zip(cols, row)) for row in rows],
indent=2), 200, {
'Content-Type': 'application/json'
})
def update_feed_desc(feed_uid, feed_desc):
"""Update a feed desc"""
feed_uid = int(feed_uid)
with dbop.db() as db:
db.execute("update fm_feeds set feed_desc=? where feed_uid=?",
[feed_desc, feed_uid])
db.commit()
def link_already(url):
with dbop.db() as db:
print >> param.activity, 'checking for deja-vu for', url,
c = db.execute("select count(*) from fm_items where item_link like ?",
[url + '%'])
l = c.fetchone()
print >> param.log, l and l[0]
return l and l[0]
def update_feed_html(feed_uid, feed_html):
"""Update a feed HTML link"""
feed_uid = int(feed_uid)
with dbop.db() as db:
db.execute("update fm_feeds set feed_html=? where feed_uid=?",
[feed_html, feed_uid])
db.commit()
def update_feed_dupcheck(feed_uid, dupcheck):
feed_uid = int(feed_uid)
dupcheck = int(bool(dupcheck))
# XXX run a dupcheck pass retroactively here if dupcheck == 1
with dbop.db() as db:
db.execute("update fm_feeds set feed_dupcheck=? where feed_uid=?",
[dupcheck, feed_uid])
db.commit()
def update_feed_title(feed_uid, feed_title):
"""Update a feed title"""
feed_uid = int(feed_uid)
with dbop.db() as db:
db.execute("update fm_feeds set feed_title=? where feed_uid=?",
[feed_title, feed_uid])
db.commit()
def hard_purge(feed_uid):
feed_uid = int(feed_uid)
with dbop.db() as db:
db.execute("delete from fm_items where item_feed_uid=?", [feed_uid])
db.execute("delete from fm_rules where rule_feed_uid=?", [feed_uid])
db.execute("delete from fm_feeds where feed_uid=?", [feed_uid])
db.commit()
filters.invalidate()
def mylos():
with dbop.db() as db:
c = db.cursor()
last = dbop.share(c)
return flask.render_template('_share.atom',
time=time,
normalize=normalize,
atom_content=atom_content,
**locals())
def rules():
with dbop.db() as db:
c = db.cursor()
feed_rules = dbop.rules(c, None)
return flask.render_template('rules.html',
filters=filters,
len=len,
max=max,
**locals())
def update_feed_exempt(feed_uid, exempt):
feed_uid = int(feed_uid)
exempt = int(bool(exempt))
with dbop.db() as db:
c = db.cursor()
db.execute("update fm_feeds set feed_exempt=? where feed_uid=?",
[exempt, feed_uid])
if exempt:
filters.exempt_feed_retroactive(db, c, feed_uid)
db.commit()
def opml():
sort_key = flask.request.args.get('sort', '(unread > 0) DESC, snr')
if sort_key == 'feed_title':
sort_key = 'lower(feed_title)'
order = flask.request.args.get('order', 'DESC')
with dbop.db() as db:
rows = dbop.opml(db)
return (flask.render_template('opml.opml',
atom_content=atom_content,
rows=rows), 200, {
'Content-Type': 'text/plain'
})
def stats():
with dbop.db() as db:
c = db.cursor()
rows = dbop.stats(c)
csvfile = cStringIO.StringIO()
out = csv.writer(csvfile, dialect='excel', delimiter=',')
out.writerow([col[0].capitalize() for col in c.description])
for row in c:
out.writerow(row)
try:
return (csvfile.getvalue(), 200, {'Content-Type': 'text/csv'})
finally:
csvfile.close()
def run():
# force loading of the database so we don't have to wait an hour to detect
# a database format issue
with dbop.db() as db:
c = db.cursor()
dbop.load_settings(c)
c.close()
logging.getLogger().setLevel(logging.INFO)
# start Flask
app.run(host=param.settings['ip'],
port=int(param.settings['port']),
threaded=True)
def update_feed_filter(feed_uid, feed_filter):
"""Update a feed desc"""
feed_uid = int(feed_uid)
feed_filter = feed_filter.strip()
if feed_filter:
# check syntax
compile(filters.normalize_rule(feed_filter), 'web form', 'eval')
val = feed_filter
else:
val = None
with dbop.db() as db:
db.execute("update fm_feeds set feed_filter=? where feed_uid=?",
[val, feed_uid])
db.commit()
filters.invalidate()
def item(uid, op):
assert op == 'edit'
status = None
if flask.request.method == 'POST':
assert check_nonce('edit%d' % uid, flask.request.form.get('nonce'))
status = update.update_item(
uid,
*[flask.request.form.get(x) for x in ['href', 'title', 'content']])
with dbop.db() as db:
title, content, href = dbop.item(db, uid)
nonce = gen_nonce('edit%d' % uid)
return flask.render_template('edit.html',
normalize=normalize,
len=len,
max=max,
**locals())
def feeds():
sort_key = flask.request.args.get('sort', '(unread > 0) DESC, snr')
if sort_key == 'feed_title':
sort_key = 'lower(feed_title)'
order = flask.request.args.get('order', 'DESC')
with dbop.db() as db:
rows = dbop.feeds(db, sort_key, order)
sum_unread = sum(int(row['unread']) for row in rows)
sum_filtered = sum(int(row['filtered']) for row in rows)
sum_interesting = sum(int(row['interesting']) for row in rows)
sum_total = sum(int(row['total']) for row in rows)
return flask.render_template('feeds.html',
since=since,
int=int,
repr=repr,
**locals())
def update(where_clause=''):
with dbop.db() as db:
c = db.cursor()
# refresh filtering rules
filters.load_rules(c)
# at 3AM by default, perform house-cleaning
if time.localtime()[3] == param.backup_hour:
cleanup(db, c)
# create worker threads and the queues used to communicate with them
work_q = Queue.Queue()
process_q = Queue.Queue()
workers = []
for i in range(param.feed_concurrency):
workers.append(FeedWorker(i + 1, work_q, process_q))
workers[-1].start()
# assign work
c.execute("""select feed_uid, feed_xml, feed_etag, feed_dupcheck,
strftime('%s', feed_modified) from fm_feeds where feed_status=0 """ +
where_clause)
for feed_uid, feed_xml, feed_etag, feed_dupcheck, feed_modified in c:
if feed_modified:
feed_modified = float(feed_modified)
feed_modified = time.localtime(feed_modified)
else:
feed_modified = None
work_q.put(
(feed_uid, feed_xml, feed_etag, feed_modified, feed_dupcheck))
# None is an indication for workers to stop
for i in range(param.feed_concurrency):
work_q.put(None)
workers_left = param.feed_concurrency
while workers_left > 0:
feed_info = process_q.get()
# exited worker
if not feed_info:
workers_left -= 1
else:
try:
update_feed(db, c, *feed_info)
except:
util.print_stack()
db.commit()
# give reader threads an opportunity to get their work done
time.sleep(1)
def settings(status=''):
op = flask.request.form.get('op', '') or flask.request.args.get('op', '')
with dbop.db() as db:
c = db.cursor()
if op == 'refresh':
__main__.updater.event.set()
status = 'Manual refresh of all feeds requested.'
elif op == 'debug':
if flask.request.form.get('debug',
'') == 'Disable verbose logging':
setattr(param, 'debug', False)
else:
setattr(param, 'debug', True)
elif op == 'facebook':
api_key = flask.request.form.get('api_key', '').strip()
if api_key:
dbop.setting(db, c, fb_api_key=api_key)
app_id = flask.request.form.get('app_id', '').strip()
if app_id:
dbop.setting(db, c, fb_app_id=app_id)
fb_secret = flask.request.form.get('fb_secret', '').strip()
if fb_secret:
dbop.setting(db, c, fb_secret=fb_secret)
elif op == 'del_token':
dbop.setting(db, c, fb_token='')
elif op == 'maint':
dbop.snr_mv(db, c)
db.commit()
stats = filters.stats(c)
return flask.render_template('settings.html',
filters=filters,
executable=sys.argv[0],
py_version=sys.version,
param_debug=param.debug,
param_settings=param.settings,
started=__main__.started,
uptime=datetime.datetime.now() -
__main__.started,
len=len,
max=max,
**locals())
def run(self):
while True:
item_uid = None
try:
item_uid, rating = self.in_q.get()
with dbop.db() as db:
c = db.cursor()
try:
c.execute(
"""update fm_items
set item_rating=?, item_rated=julianday('now')
where item_uid=?""", [rating, item_uid])
fb_token = param.settings.get('fb_token', None)
if rating == 1 and fb_token:
c.execute(
"""select feed_uid, item_link, item_title, feed_private
from fm_items, fm_feeds
where item_uid=? and feed_uid=item_feed_uid""", [item_uid])
feed_uid, url, title, private = c.fetchone()
db.commit()
if rating == 1 and fb_token and not private:
callout = random.choice([
'Interesting: ', 'Notable: ', 'Recommended: ',
'Thumbs-up: ', 'Noteworthy: ', 'FYI: ',
'Worth reading: '
])
try:
social.fb_post(fb_token, callout + title, url)
except social.ExpiredToken:
notification(
db,
c,
feed_uid,
'Service notification',
'The Facebook access token has expired',
link='/settings#facebook')
except:
util.print_stack()
except:
util.print_stack()
if item_uid is not None:
self.in_q.put((item_uid, rating))
def import_opml(opml_file):
tree = parse_opml(opml_file)
with dbop.db() as db:
c = db.cursor()
ok = 0
dup = 0
for feed in tree:
feed['feed_etag'] = ''
try:
c.execute("""insert into fm_feeds
(feed_xml, feed_etag, feed_html, feed_title, feed_desc) values
(:xmlUrl, :feed_etag, :htmlUrl, :title, :desc)""", feed)
ok += 1
except sqlite.IntegrityError, e:
if 'feed_xml' not in str(e):
raise
dup += 1
db.commit()
print ok, 'feeds imported,', dup, 'rejected as duplicates'
def add_feed():
if flask.request.method == 'POST':
feed_xml = flask.request.form.get('feed_xml', '').strip()
if feed_xml:
with dbop.db() as db:
c = db.cursor()
try:
feed_uid, feed_title, num_added, num_filtered \
= update.add_feed(feed_xml)
except update.ParseError:
feed_err = 'Connection or parse error in subcription attempt.'
resolution = 'check URL'
except update.AutodiscoveryParseError:
feed_err = 'Autodiscovery failed.'
resolution = 'you need to find a valid feed URL'
except update.FeedAlreadyExists:
feed_err = 'The feed URL is already assigned to another feed.'
resolution = 'check for duplicates'
except requests.exceptions.RequestException as e:
feed_err = 'Error loading URL during autodiscovery attempt: %r' % e
except update.UnknownError, e:
feed_err = 'Unknown error: %r' % e.args[0]