def insert_item(item):
"""docstring for item_insert"""
frag = fragments_fromstring(item.content)
for e in frag:
if not type(e) == HtmlElement:
continue
for i in e.xpath('//img'):
f = '/' + i.attrib['src'].replace('http://', '').replace(
'/', '_').split('?')[0]
if exists(cfg.get('cache', 'dir') + f):
continue
try:
urllib.urlretrieve(i.attrib['src'],
cfg.get('cache', 'dir') + f)
i.attrib['src'] = cfg.get('cache', 'url') + f
except:
# TODO
pass
c = ''
for i in frag:
if type(i) == HtmlElement:
c += unicode(tostring(i))
else:
c += i
item.content = c
return item
def insert_item(item):
"""docstring for item_insert"""
frag = fragments_fromstring(item.content)
for e in frag:
if not type(e) == HtmlElement:
continue
for i in e.xpath('//img'):
f = '/'+i.attrib['src'].replace('http://', '').replace('/','_').split('?')[0]
if exists(cfg.get('cache', 'dir')+f):
continue
try:
urllib.urlretrieve(i.attrib['src'], cfg.get('cache', 'dir')+f)
i.attrib['src'] = cfg.get('cache', 'url')+f
except:
# TODO
pass
c = ''
for i in frag:
if type(i) == HtmlElement:
c += unicode(tostring(i))
else:
c += i
item.content = c
return item
def top():
limit = int(cfg.get('app', 'items_per_page'))
items = Item.query.filter(Item.archived==False).order_by(Item.added).limit(limit).all()
return render_template('flat.html'
,items = items
,unarchiveds = get_unarchived_ids(items)
)
def all(page_num=1):
limit = int(cfg.get('app', 'items_per_page'))
offset = limit*(page_num-1)
items = Item.query.order_by(Item.added).limit(limit).offset(offset).all()
pagination = Pagination(page_num, limit, Item.query.count())
return render_template('flat.html'
,pagination = pagination
,items = items
,unarchiveds = get_unarchived_ids(items)
,menu_path= '/all'
)
def top(page_num=1):
limit = int(cfg.get('app', 'items_per_page'))
offset = limit*(page_num-1)
items = Item.query.filter(Item.archived==False).order_by(Item.added).limit(limit).offset(offset).all()
pagination = Pagination(page_num, limit, Item.query.filter(Item.archived==False).count())
return render_template('flat.html'
,items = items
,pagination = pagination
,unarchiveds = get_unarchived_ids(items)
,menu_path = '/top' #preserve menu highlight when paging
)
def top(page_num=1):
limit = int(cfg.get('app', 'items_per_page'))
offset = limit*(page_num-1)
items = Item.query.filter(Item.archived==False).order_by(Item.added).limit(limit).offset(offset).all()
pagination = Pagination(page_num, limit, Item.query.filter(Item.archived==False).count())
return render_template('flat.html'
,items = items
,pagination = pagination
,unarchiveds = get_unarchived_ids(items)
,menu_path = '/top' #preserve menu highlight when paging
)
def all(page_num=1):
limit = int(cfg.get('app', 'items_per_page'))
offset = limit*(page_num-1)
items = Item.query.order_by(Item.added).limit(limit).offset(offset).all()
pagination = Pagination(page_num, limit, Item.query.count())
return render_template('flat.html'
,pagination = pagination
,items = items
,unarchiveds = get_unarchived_ids(items)
,menu_path= '/all'
)
def do_query(q_str):
page_num = 1
if(q_str.find('/')):
try:
page_num = int(q_str.split('/')[-1])
q_str = ''.join(q_str.split('/')[:-1])
except:
pass
reverse = False
if(q_str.startswith('!')):
q_str = q_str[1:]
reverse = True
rules = q_str.split(',')
query = db.session.query(Item).filter(Item.source_id==Source.source_id)
for rule in rules:
if rule.find(':') != -1:
item, value = rule.split(':', 1)
if item.startswith('~'):
query = query.filter(getattr(Item, item[1:]).contains(value))
elif item.startswith('-'):
query = query.filter(not_(getattr(Item, item[1:]).contains(value)))
else:
query = query.filter(getattr(Item, item) == value)
continue
if rule.startswith('_'):
query = query.filter(Source.name == rule[1:])
continue
count = query.count()
limit = int(cfg.get('app', 'items_per_page'))
offset = limit*(page_num-1)
items = query.limit(limit).offset(offset).all()
if reverse:
items.reverse()
pagination = Pagination(page_num, limit, count)
return render_template('flat.html'
,pagination = pagination
,items = items
,unarchiveds = get_unarchived_ids(items)
,menu_path = '/query/%s' % q_str
)
def do_query(q_str):
page_num = 1
if(q_str.find('/')):
try:
page_num = int(q_str.split('/')[-1])
q_str = ''.join(q_str.split('/')[:-1])
except:
pass
#if(q_str.startswith('!')):
# q_str = q_str[1:]
# reverse = True
rules = q_str.split(',')
query = db_session.query(Item).filter(Item.source_id==Source.source_id)
for rule in rules:
if rule.find(':') != -1:
item, value = rule.split(':', 1)
if item.startswith('~'):
query = query.filter(getattr(Item, item[1:]).contains(value))
elif item.startswith('-'):
query = query.filter(not_(getattr(Item, item[1:]).contains(value)))
else:
query = query.filter(getattr(Item, item) == value)
continue
if rule.startswith('_'):
query = query.filter(Source.name == rule[1:])
continue
count = query.count()
limit = int(cfg.get('app', 'items_per_page'))
offset = limit*(page_num-1)
items = query.limit(limit).offset(offset).all()
#if reverse:
# items.reverse()
pagination = Pagination(page_num, limit, count)
return render_template('flat.html'
,pagination = pagination
,items = items
,unarchiveds = get_unarchived_ids(items)
,menu_path = '/query/%s' % q_str
)
# merchantability or fitness for a particular purpose. see the
# gnu affero general public license for more details.
#
# you should have received a copy of the gnu affero general public license
# along with potion. if not, see <http://www.gnu.org/licenses/>.
#
# (c) 2012- by adam tauber, <*****@*****.**>
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Text, DateTime, Boolean, PickleType
from sqlalchemy.orm import relationship, scoped_session, sessionmaker, backref
from sqlalchemy.ext.declarative import declarative_base
from datetime import datetime
from pickle import dumps, loads
from potion.common import cfg
engine = create_engine(cfg.get("database", "connection"))
db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine))
Base = declarative_base()
Base.metadata.bind = engine
Base.query = db_session.query_property()
class User(Base):
__tablename__ = "users"
user_id = Column(Integer, primary_key=True)
name = Column(String(511))
password = Column(String(511))
is_active = Column(Boolean)
import re
from feedparser import parse
from datetime import datetime
from urlparse import urlparse, urlunparse
from itertools import ifilterfalse, imap
import urllib
import urllib2
import httplib
from lxml import etree
from io import StringIO
from potion.common import cfg
from potion.models import db_session, Source, Item
user_agent = cfg.get('fetcher', 'user_agent')
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', user_agent)]
#allowed self_close tags
allowed_self_close_tags = ('area', 'base', 'basefont', 'br', 'hr', 'input',
'img', 'link', 'meta')
# removes annoying UTM params to urls.
utmRe = re.compile('utm_(source|medium|campaign|content)=')
def urlSanitize(url):
# handle any redirected urls from the feed, like
# ('http://feedproxy.google.com/~r/Torrentfreak/~3/8UY1UySQe1k/')
# You should have received a copy of the GNU Affero General Public License
# along with potion. If not, see <http://www.gnu.org/licenses/>.
#
# (C) 2012- by Adam Tauber, <*****@*****.**>
from flask import Flask, request, render_template, redirect, flash
from potion.models import db_session, Item, Source, Query
from potion.common import cfg
from flaskext.wtf import Form, TextField, Required, SubmitField
menu_items = (('/' , 'home')
,('/doc' , 'documentation')
,('/add/source' , 'add source')
,('/queries' , 'queries')
,('/top' , 'top '+cfg.get('app', 'items_per_page'))
,('/all' , 'all')
)
app = Flask(__name__)
app.secret_key = cfg.get('app', 'secret_key')
class SourceForm(Form):
#name, address, source_type, is_public=True, attributes={}
name = TextField('Name' , [Required()])
source_type = TextField('Type' , [Required()])
address = TextField('Address' , [Required()])
submit = SubmitField('Submit' , [Required()])
@app.context_processor
from flask import request, render_template, redirect, flash
from sqlalchemy import not_
from potion.models import Item, Source, Query
from potion.common import cfg
from flask_wtf import Form
from wtforms.fields import TextField, SubmitField
from wtforms.validators import Required
from potion.helpers import Pagination
from potion import app, db
menu_items = (('/' , 'home')
#,('/doc' , 'documentation')
,('/sources' , 'sources')
,('/queries' , 'queries')
,('/top' , 'top %s unarchived' % cfg.get('app', 'items_per_page'))
,('/all' , 'all')
)
class SourceForm(Form):
#name, address, source_type, is_public=True, attributes={}
name = TextField('Name' , [Required()])
source_type = TextField('Type' , [Required()])
address = TextField('Address' , [Required()])
submit = SubmitField('Submit' , [Required()])
class QueryForm(Form):
name = TextField('Name' , [Required()])
query_string = TextField('Query', [Required()])
submit = SubmitField('Submit' , [Required()])
from feedparser import parse
from datetime import datetime
from urlparse import urlparse, urlunparse
from itertools import ifilterfalse, imap
import urllib
import urllib2
import httplib
from lxml import etree
from io import StringIO
from potion.common import cfg
from potion.models import db_session, Source, Item
from potion import proxy
user_agent = cfg.get("fetcher", "user_agent")
opener = urllib2.build_opener()
opener.addheaders = [("User-agent", user_agent)]
# allowed self_close tags
allowed_self_close_tags = ("area", "base", "basefont", "br", "hr", "input", "img", "link", "meta")
# removes annoying UTM params to urls.
utmRe = re.compile("utm_(source|medium|campaign|content)=")
def urlSanitize(url):
# handle any redirected urls from the feed, like
# ('http://feedproxy.google.com/~r/Torrentfreak/~3/8UY1UySQe1k/')
us = httplib.urlsplit(url)
import re
from feedparser import parse
from datetime import datetime
from urlparse import urlparse, urlunparse
from itertools import ifilterfalse, imap
import urllib
import urllib2
import httplib
from potion.common import cfg
from potion.models import Source, Item
from potion import proxy, db
user_agent = cfg.get('fetcher', 'user_agent')
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', user_agent)]
# removes annoying UTM params to urls.
utmRe=re.compile('utm_(source|medium|campaign|content)=')
def urlSanitize(url):
# handle any redirected urls from the feed, like
# ('http://feedproxy.google.com/~r/Torrentfreak/~3/8UY1UySQe1k/')
us=httplib.urlsplit(url)
if us.scheme=='http':
conn = httplib.HTTPConnection(us.netloc, timeout=3)
req = urllib.quote(url[7+len(us.netloc):])
elif us.scheme=='https':
conn = httplib.HTTPSConnection(us.netloc)
from os.path import realpath, dirname
path.append(realpath(dirname(realpath(__file__))+'/../'))
from flask import Flask, request, render_template, redirect, flash
from sqlalchemy import not_
from potion.models import db_session, Item, Source, Query
from potion.common import cfg
from flask.ext.wtf import Form, TextField, Required, SubmitField
from potion.helpers import Pagination
menu_items = (('/' , 'home')
#,('/doc' , 'documentation')
,('/sources' , 'sources')
,('/queries' , 'queries')
,('/top' , 'top %s unarchived' % cfg.get('app', 'items_per_page'))
,('/all' , 'all')
)
app = Flask(__name__)
app.secret_key = cfg.get('app', 'secret_key')
class SourceForm(Form):
#name, address, source_type, is_public=True, attributes={}
name = TextField('Name' , [Required()])
source_type = TextField('Type' , [Required()])
address = TextField('Address' , [Required()])
submit = SubmitField('Submit' , [Required()])
@app.context_processor
#
# (c) 2012- by adam tauber, <*****@*****.**>
if __name__ == '__main__':
from sys import path
from os.path import realpath, dirname
path.append(realpath(dirname(realpath(__file__)) + '/../../'))
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Text, DateTime, Boolean, PickleType
from sqlalchemy.orm import relationship, scoped_session, sessionmaker, backref
from sqlalchemy.ext.declarative import declarative_base
from datetime import datetime
from pickle import dumps, loads
from potion.common import cfg
engine = create_engine(cfg.get('database', 'connection'))
db_session = scoped_session(
sessionmaker(autocommit=False, autoflush=False, bind=engine))
Base = declarative_base()
Base.metadata.bind = engine
Base.query = db_session.query_property()
class User(Base):
__tablename__ = 'users'
user_id = Column(Integer, primary_key=True)
name = Column(String(511))
password = Column(String(511))
is_active = Column(Boolean)
import urllib2
from potion.common import cfg
if cfg.has_section('proxy') and cfg.has_option('proxy','proxy_type') and cfg.has_option('proxy','proxy_address'):
proxy_type = cfg.get('proxy', 'proxy_type')
proxy_address = cfg.get('proxy', 'proxy_address')
import socks
(proxy_host,proxy_port)=proxy_address.split(':')
try:
proxy_port=int(proxy_port)
except ValueError:
proxy_port=None
print '[EE]proxy port must be int!'
if not proxy_host or not proxy_port:
#maybe mispelled config, exiting with code 'incorrect usage'
print '[EE]Error in proxy address configuration(current value: %s)!' % proxy_address
import sys
sys.exit(2)
socks.setdefaultproxy(eval('socks.PROXY_TYPE_'+proxy_type), proxy_host, int(proxy_port))
socks.wrapmodule(urllib2)
from flask import Flask, request, render_template, redirect, flash
from sqlalchemy import not_
from potion.models import db_session, Item, Source, Query
from potion.common import cfg
from flask.ext.wtf import Form
from wtforms import TextField, SubmitField
from wtforms.validators import Required
from potion.helpers import Pagination
menu_items = (('/' , 'home')
#,('/doc' , 'documentation')
,('/sources' , 'sources')
,('/queries' , 'queries')
,('/top' , 'top %s unarchived' % cfg.get('app', 'items_per_page'))
,('/saved' , 'top %s saved' % cfg.get('app', 'items_per_page'))
,('/all' , 'all')
)
app = Flask(__name__)
app.secret_key = cfg.get('app', 'secret_key')
class SourceForm(Form):
#name, address, source_type, is_public=True, attributes={}
name = TextField('Name' , [Required()])
source_type = TextField('Type' , [Required()])
address = TextField('Address' , [Required()])
submit = SubmitField('Submit' , [Required()])
#
# (c) 2012- by adam tauber, <*****@*****.**>
if __name__ == '__main__':
from sys import path
from os.path import realpath, dirname
path.append(realpath(dirname(realpath(__file__))+'/../../'))
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Text, DateTime, Boolean, PickleType
from sqlalchemy.orm import relationship, scoped_session, sessionmaker, backref
from sqlalchemy.ext.declarative import declarative_base
from datetime import datetime
from pickle import dumps, loads
from potion.common import cfg
engine = create_engine(cfg.get('database', 'connection'))
db_session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=engine))
Base = declarative_base()
Base.metadata.bind = engine
Base.query = db_session.query_property()
class User(Base):
__tablename__ = 'users'
user_id = Column(Integer, primary_key=True)
name = Column(String(511))
password = Column(String(511))
is_active = Column(Boolean)
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
from potion.common import cfg
app = Flask(__name__)
app.secret_key = cfg.get('app', 'secret_key')
app.config['SQLALCHEMY_DATABASE_URI'] = cfg.get('database', 'connection')
db = SQLAlchemy(app)
