def save(save_id=0):
db_session.query(Item).filter(Item.item_id==save_id).update({Item.saved: True}, synchronize_session='fetch')
db_session.commit()
if save_id:
return render_template('status.html', messages=['item(%s) saved' % save_id])
flash('Successfully saved item')
return redirect(request.referrer or '/')
def archive():
try:
ids = map(int, request.form.get('ids', '').split(','))
except:
flash('Bad params')
return redirect(request.referrer or '/')
db_session.query(Item).filter(Item.item_id.in_(ids)).update({Item.archived: True}, synchronize_session='fetch')
db_session.commit()
flash('Successfully archived items: %d' % len(ids))
return redirect(request.referrer or '/')
def add(source_name, item_name, content, url=None, attributes=None):
if not attributes: attributes = {}
source = db_session.query(Source).filter(Source.name==source_name).first() or Source(name=source_name, source_type='cmdline', address='/query/_%s' % source_name)
item = Item(item_name, content, url=url, attributes=attributes)
db_session.add(source)
source.items.append(item)
db_session.commit()
return item
def archive(id=0):
if request.method=='POST':
try:
ids = map(int, request.form.get('ids', '').split(','))
except:
flash('Bad params')
return redirect(request.referrer or '/')
elif id==0:
flash('Nothing to archive')
return redirect(request.referrer or '/')
else:
ids=[id]
db_session.query(Item).filter(Item.item_id.in_(ids)).update({Item.archived: True}, synchronize_session='fetch')
db_session.commit()
if id:
return render_template('status.html', messages=['item(%s) archived' % id])
flash('Successfully archived items: %d' % len(ids))
return redirect(request.referrer or '/')
def parseFeed(feed):
counter = 0
#modified = feed['modified'].timetuple() if feed.get('modified') else None
f = None
f = parse(fetchFeed(feed.address)
,etag = feed.attributes.get('etag')
,modified = feed.attributes.get('modified')
)
if not f:
print '[EE] cannot parse %s - %s' % (feed.name, feed.address)
return counter
#print '[!] parsing %s - %s' % (feed.name, feed.url)
try:
feed.attributes['etag'] = f.etag
except AttributeError:
pass
try:
feed.attributes['modified'] = f.modified
except AttributeError:
pass
d = feed.updated
for item in reversed(f['entries']):
try:
u = urlSanitize(item['links'][0]['href'])
except:
u = ''
# checking duplications
if db_session.query(Item).filter(Item.source_id==feed.source_id).filter(Item.url==u).first():
continue
try:
tmp_date = datetime(*item['updated_parsed'][:6])
except:
tmp_date = datetime.now()
# title content updated
try:
c = unicode(''.join([x.value for x in item.content]))
except:
c = u'[EE] No content found, plz check the feed (%s) and fix me' % feed.name
for key in ['media_text', 'summary', 'description', 'media:description']:
if item.has_key(key):
c = unicode(item[key])
break
t = unicode(item.get('title','[EE] Notitle'))
# date as tmp_date?!
feed.items.append(Item(t, c, url=u, attributes={'date':tmp_date}))
db_session.commit()
counter += 1
feed.updated = d
db_session.commit()
#feed.save()
return counter
def add(source_name, item_name, content, url=None, attributes=None):
if not attributes: attributes = {}
source = db_session.query(Source).filter(
Source.name == source_name).first() or Source(
name=source_name,
source_type='cmdline',
address='/query/_%s' % source_name)
item = Item(item_name, content, url=url, attributes=attributes)
db_session.add(source)
source.items.append(item)
db_session.commit()
return item
def do_query(q_str):
page_num = 1
if(q_str.find('/')):
try:
page_num = int(q_str.split('/')[-1])
q_str = ''.join(q_str.split('/')[:-1])
except:
pass
#if(q_str.startswith('!')):
# q_str = q_str[1:]
# reverse = True
rules = q_str.split(',')
query = db_session.query(Item).filter(Item.source_id==Source.source_id)
for rule in rules:
if rule.find(':') != -1:
item, value = rule.split(':', 1)
if item.startswith('~'):
query = query.filter(getattr(Item, item[1:]).contains(value))
elif item.startswith('-'):
query = query.filter(not_(getattr(Item, item[1:]).contains(value)))
else:
query = query.filter(getattr(Item, item) == value)
continue
if rule.startswith('_'):
query = query.filter(Source.name == rule[1:])
continue
count = query.count()
limit = int(cfg.get('app', 'items_per_page'))
offset = limit*(page_num-1)
items = query.limit(limit).offset(offset).all()
#if reverse:
# items.reverse()
pagination = Pagination(page_num, limit, count)
return render_template('flat.html'
,pagination = pagination
,items = items
,unarchiveds = get_unarchived_ids(items)
,menu_path = '/query/%s' % q_str
)
def parseFeed(feed):
counter = 0
# modified = feed['modified'].timetuple() if feed.get('modified') else None
f = None
f = parse(fetchFeed(feed.address), etag=feed.attributes.get("etag"), modified=feed.attributes.get("modified"))
if not f:
print "[EE] cannot parse %s - %s" % (feed.name, feed.address)
return counter
# print '[!] parsing %s - %s' % (feed.name, feed.url)
try:
if feed.attributes["etag"] != f.etag:
return
except KeyError:
pass
try:
feed.attributes["etag"] = f.etag
except AttributeError:
pass
try:
feed.attributes["modified"] = f.modified
except AttributeError:
pass
d = feed.updated
for item in reversed(f["entries"]):
if "links" in item:
original_url = unicode(item["links"][0]["href"])
else:
original_url = unicode(item["link"])
# checking duplications
if (
db_session.query(Item)
.filter(Item.source_id == feed.source_id)
.filter(Item.original_url == original_url)
.first()
):
continue
try:
u = urlSanitize(original_url)
except:
u = ""
try:
tmp_date = datetime(*item["updated_parsed"][:6])
except:
tmp_date = datetime.now()
# title content updated
try:
c = "".join([x.value for x in item.content])
except:
c = u"[EE] No content found, plz check the feed (%s) and fix me" % feed.name
for key in ["media_text", "summary", "description", "media:description"]:
if item.has_key(key):
c = item[key]
break
# fixing malformed html
if c:
original = c
c = ""
try:
phtml = etree.parse(StringIO(original), etree.HTMLParser())
for node in phtml.iter("*"):
clean_description(node)
for node in phtml.xpath("//body/*"):
c += etree.tostring(node)
except:
print u"[EE]description parsing error(%s - %s)" % (feed.name, u)
c = original
t = item.get("title", "[EE] Notitle")
# date as tmp_date?!
feed.items.append(Item(t, c, original_url, url=u, attributes={"date": tmp_date}))
db_session.commit()
counter += 1
feed.updated = d
db_session.commit()
# feed.save()
return counter
def parseFeed(feed):
counter = 0
#modified = feed['modified'].timetuple() if feed.get('modified') else None
f = None
f = parse(fetchFeed(feed.address)
,etag = feed.attributes.get('etag')
,modified = feed.attributes.get('modified')
)
if not f:
print '[EE] cannot parse %s - %s' % (feed.name, feed.address)
return counter
#print '[!] parsing %s - %s' % (feed.name, feed.url)
try:
if feed.attributes['etag'] != f.etag:
return
except KeyError:
pass
try:
feed.attributes['etag'] = f.etag
except AttributeError:
pass
try:
feed.attributes['modified'] = f.modified
except AttributeError:
pass
d = feed.updated
for item in reversed(f['entries']):
if 'links' in item:
original_url = unicode(item['links'][0]['href'])
else:
original_url = unicode(item['link'])
# checking duplications
if db_session.query(Item). \
filter(Item.source_id==feed.source_id). \
filter(Item.original_url==original_url).first():
continue
try:
u = urlSanitize(original_url)
except:
u = ''
try:
tmp_date = datetime(*item['updated_parsed'][:6])
except:
tmp_date = datetime.now()
# title content updated
try:
c = ''.join([x.value for x in item.content])
except:
c = u'[EE] No content found, plz check the feed (%s) and fix me' % feed.name
for key in ['media_text', 'summary', 'description', 'media:description']:
if item.has_key(key):
c = item[key]
break
# #fixing malformed html
# if c:
# original = c
# c = ''
# try:
# phtml = etree.parse(StringIO(original), etree.HTMLParser())
# for node in phtml.iter('*'):
# clean_description(node)
# for node in phtml.xpath('//body/*'):
# c += etree.tostring(node)
# except:
# print u'[EE]description parsing error(%s - %s)' % (feed.name, u)
# c = original
t = item.get('title','[EE] Notitle')
# date as tmp_date?!
feed.items.append(Item(t, c, original_url, url=u, attributes={'date':tmp_date}))
db_session.commit()
counter += 1
feed.updated = d
db_session.commit()
#feed.save()
return counter
def parseFeed(feed):
counter = 0
#modified = feed['modified'].timetuple() if feed.get('modified') else None
f = None
f = parse(fetchFeed(feed.address),
etag=feed.attributes.get('etag'),
modified=feed.attributes.get('modified'))
if not f:
print '[EE] cannot parse %s - %s' % (feed.name, feed.address)
return counter
#print '[!] parsing %s - %s' % (feed.name, feed.url)
try:
if feed.attributes['etag'] != f.etag:
return
except KeyError:
pass
try:
feed.attributes['etag'] = f.etag
except AttributeError:
pass
try:
feed.attributes['modified'] = f.modified
except AttributeError:
pass
d = feed.updated
for item in reversed(f['entries']):
if 'links' in item:
original_url = unicode(item['links'][0]['href'])
else:
original_url = unicode(item['link'])
# checking duplications
if db_session.query(Item). \
filter(Item.source_id==feed.source_id). \
filter(Item.original_url==original_url).first():
continue
try:
u = urlSanitize(original_url)
except:
u = ''
try:
tmp_date = datetime(*item['updated_parsed'][:6])
except:
tmp_date = datetime.now()
# title content updated
try:
c = ''.join([x.value for x in item.content])
except:
c = u'[EE] No content found, plz check the feed (%s) and fix me' % feed.name
for key in [
'media_text', 'summary', 'description', 'media:description'
]:
if item.has_key(key):
c = item[key]
break
# #fixing malformed html
# if c:
# original = c
# c = ''
# try:
# phtml = etree.parse(StringIO(original), etree.HTMLParser())
# for node in phtml.iter('*'):
# clean_description(node)
# for node in phtml.xpath('//body/*'):
# c += etree.tostring(node)
# except:
# print u'[EE]description parsing error(%s - %s)' % (feed.name, u)
# c = original
t = item.get('title', '[EE] Notitle')
# date as tmp_date?!
feed.items.append(
Item(t, c, original_url, url=u, attributes={'date': tmp_date}))
db_session.commit()
counter += 1
feed.updated = d
db_session.commit()
#feed.save()
return counter
