def insert_item(item): """docstring for item_insert""" frag = fragments_fromstring(item.content) for e in frag: if not type(e) == HtmlElement: continue for i in e.xpath('//img'): f = '/' + i.attrib['src'].replace('http://', '').replace( '/', '_').split('?')[0] if exists(cfg.get('cache', 'dir') + f): continue try: urllib.urlretrieve(i.attrib['src'], cfg.get('cache', 'dir') + f) i.attrib['src'] = cfg.get('cache', 'url') + f except: # TODO pass c = '' for i in frag: if type(i) == HtmlElement: c += unicode(tostring(i)) else: c += i item.content = c return item
def insert_item(item): """docstring for item_insert""" frag = fragments_fromstring(item.content) for e in frag: if not type(e) == HtmlElement: continue for i in e.xpath('//img'): f = '/'+i.attrib['src'].replace('http://', '').replace('/','_').split('?')[0] if exists(cfg.get('cache', 'dir')+f): continue try: urllib.urlretrieve(i.attrib['src'], cfg.get('cache', 'dir')+f) i.attrib['src'] = cfg.get('cache', 'url')+f except: # TODO pass c = '' for i in frag: if type(i) == HtmlElement: c += unicode(tostring(i)) else: c += i item.content = c return item
def top(): limit = int(cfg.get('app', 'items_per_page')) items = Item.query.filter(Item.archived==False).order_by(Item.added).limit(limit).all() return render_template('flat.html' ,items = items ,unarchiveds = get_unarchived_ids(items) )
def all(page_num=1): limit = int(cfg.get('app', 'items_per_page')) offset = limit*(page_num-1) items = Item.query.order_by(Item.added).limit(limit).offset(offset).all() pagination = Pagination(page_num, limit, Item.query.count()) return render_template('flat.html' ,pagination = pagination ,items = items ,unarchiveds = get_unarchived_ids(items) ,menu_path= '/all' )
def top(page_num=1): limit = int(cfg.get('app', 'items_per_page')) offset = limit*(page_num-1) items = Item.query.filter(Item.archived==False).order_by(Item.added).limit(limit).offset(offset).all() pagination = Pagination(page_num, limit, Item.query.filter(Item.archived==False).count()) return render_template('flat.html' ,items = items ,pagination = pagination ,unarchiveds = get_unarchived_ids(items) ,menu_path = '/top' #preserve menu highlight when paging )
def do_query(q_str): page_num = 1 if(q_str.find('/')): try: page_num = int(q_str.split('/')[-1]) q_str = ''.join(q_str.split('/')[:-1]) except: pass reverse = False if(q_str.startswith('!')): q_str = q_str[1:] reverse = True rules = q_str.split(',') query = db.session.query(Item).filter(Item.source_id==Source.source_id) for rule in rules: if rule.find(':') != -1: item, value = rule.split(':', 1) if item.startswith('~'): query = query.filter(getattr(Item, item[1:]).contains(value)) elif item.startswith('-'): query = query.filter(not_(getattr(Item, item[1:]).contains(value))) else: query = query.filter(getattr(Item, item) == value) continue if rule.startswith('_'): query = query.filter(Source.name == rule[1:]) continue count = query.count() limit = int(cfg.get('app', 'items_per_page')) offset = limit*(page_num-1) items = query.limit(limit).offset(offset).all() if reverse: items.reverse() pagination = Pagination(page_num, limit, count) return render_template('flat.html' ,pagination = pagination ,items = items ,unarchiveds = get_unarchived_ids(items) ,menu_path = '/query/%s' % q_str )
def do_query(q_str): page_num = 1 if(q_str.find('/')): try: page_num = int(q_str.split('/')[-1]) q_str = ''.join(q_str.split('/')[:-1]) except: pass #if(q_str.startswith('!')): # q_str = q_str[1:] # reverse = True rules = q_str.split(',') query = db_session.query(Item).filter(Item.source_id==Source.source_id) for rule in rules: if rule.find(':') != -1: item, value = rule.split(':', 1) if item.startswith('~'): query = query.filter(getattr(Item, item[1:]).contains(value)) elif item.startswith('-'): query = query.filter(not_(getattr(Item, item[1:]).contains(value))) else: query = query.filter(getattr(Item, item) == value) continue if rule.startswith('_'): query = query.filter(Source.name == rule[1:]) continue count = query.count() limit = int(cfg.get('app', 'items_per_page')) offset = limit*(page_num-1) items = query.limit(limit).offset(offset).all() #if reverse: # items.reverse() pagination = Pagination(page_num, limit, count) return render_template('flat.html' ,pagination = pagination ,items = items ,unarchiveds = get_unarchived_ids(items) ,menu_path = '/query/%s' % q_str )
# merchantability or fitness for a particular purpose. see the # gnu affero general public license for more details. # # you should have received a copy of the gnu affero general public license # along with potion. if not, see <http://www.gnu.org/licenses/>. # # (c) 2012- by adam tauber, <*****@*****.**> from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Text, DateTime, Boolean, PickleType from sqlalchemy.orm import relationship, scoped_session, sessionmaker, backref from sqlalchemy.ext.declarative import declarative_base from datetime import datetime from pickle import dumps, loads from potion.common import cfg engine = create_engine(cfg.get("database", "connection")) db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) Base = declarative_base() Base.metadata.bind = engine Base.query = db_session.query_property() class User(Base): __tablename__ = "users" user_id = Column(Integer, primary_key=True) name = Column(String(511)) password = Column(String(511)) is_active = Column(Boolean)
import re from feedparser import parse from datetime import datetime from urlparse import urlparse, urlunparse from itertools import ifilterfalse, imap import urllib import urllib2 import httplib from lxml import etree from io import StringIO from potion.common import cfg from potion.models import db_session, Source, Item user_agent = cfg.get('fetcher', 'user_agent') opener = urllib2.build_opener() opener.addheaders = [('User-agent', user_agent)] #allowed self_close tags allowed_self_close_tags = ('area', 'base', 'basefont', 'br', 'hr', 'input', 'img', 'link', 'meta') # removes annoying UTM params to urls. utmRe = re.compile('utm_(source|medium|campaign|content)=') def urlSanitize(url): # handle any redirected urls from the feed, like # ('http://feedproxy.google.com/~r/Torrentfreak/~3/8UY1UySQe1k/')
# You should have received a copy of the GNU Affero General Public License # along with potion. If not, see <http://www.gnu.org/licenses/>. # # (C) 2012- by Adam Tauber, <*****@*****.**> from flask import Flask, request, render_template, redirect, flash from potion.models import db_session, Item, Source, Query from potion.common import cfg from flaskext.wtf import Form, TextField, Required, SubmitField menu_items = (('/' , 'home') ,('/doc' , 'documentation') ,('/add/source' , 'add source') ,('/queries' , 'queries') ,('/top' , 'top '+cfg.get('app', 'items_per_page')) ,('/all' , 'all') ) app = Flask(__name__) app.secret_key = cfg.get('app', 'secret_key') class SourceForm(Form): #name, address, source_type, is_public=True, attributes={} name = TextField('Name' , [Required()]) source_type = TextField('Type' , [Required()]) address = TextField('Address' , [Required()]) submit = SubmitField('Submit' , [Required()]) @app.context_processor
from flask import request, render_template, redirect, flash from sqlalchemy import not_ from potion.models import Item, Source, Query from potion.common import cfg from flask_wtf import Form from wtforms.fields import TextField, SubmitField from wtforms.validators import Required from potion.helpers import Pagination from potion import app, db menu_items = (('/' , 'home') #,('/doc' , 'documentation') ,('/sources' , 'sources') ,('/queries' , 'queries') ,('/top' , 'top %s unarchived' % cfg.get('app', 'items_per_page')) ,('/all' , 'all') ) class SourceForm(Form): #name, address, source_type, is_public=True, attributes={} name = TextField('Name' , [Required()]) source_type = TextField('Type' , [Required()]) address = TextField('Address' , [Required()]) submit = SubmitField('Submit' , [Required()]) class QueryForm(Form): name = TextField('Name' , [Required()]) query_string = TextField('Query', [Required()]) submit = SubmitField('Submit' , [Required()])
from feedparser import parse from datetime import datetime from urlparse import urlparse, urlunparse from itertools import ifilterfalse, imap import urllib import urllib2 import httplib from lxml import etree from io import StringIO from potion.common import cfg from potion.models import db_session, Source, Item from potion import proxy user_agent = cfg.get("fetcher", "user_agent") opener = urllib2.build_opener() opener.addheaders = [("User-agent", user_agent)] # allowed self_close tags allowed_self_close_tags = ("area", "base", "basefont", "br", "hr", "input", "img", "link", "meta") # removes annoying UTM params to urls. utmRe = re.compile("utm_(source|medium|campaign|content)=") def urlSanitize(url): # handle any redirected urls from the feed, like # ('http://feedproxy.google.com/~r/Torrentfreak/~3/8UY1UySQe1k/') us = httplib.urlsplit(url)
import re from feedparser import parse from datetime import datetime from urlparse import urlparse, urlunparse from itertools import ifilterfalse, imap import urllib import urllib2 import httplib from potion.common import cfg from potion.models import Source, Item from potion import proxy, db user_agent = cfg.get('fetcher', 'user_agent') opener = urllib2.build_opener() opener.addheaders = [('User-agent', user_agent)] # removes annoying UTM params to urls. utmRe=re.compile('utm_(source|medium|campaign|content)=') def urlSanitize(url): # handle any redirected urls from the feed, like # ('http://feedproxy.google.com/~r/Torrentfreak/~3/8UY1UySQe1k/') us=httplib.urlsplit(url) if us.scheme=='http': conn = httplib.HTTPConnection(us.netloc, timeout=3) req = urllib.quote(url[7+len(us.netloc):]) elif us.scheme=='https': conn = httplib.HTTPSConnection(us.netloc)
from os.path import realpath, dirname path.append(realpath(dirname(realpath(__file__))+'/../')) from flask import Flask, request, render_template, redirect, flash from sqlalchemy import not_ from potion.models import db_session, Item, Source, Query from potion.common import cfg from flask.ext.wtf import Form, TextField, Required, SubmitField from potion.helpers import Pagination menu_items = (('/' , 'home') #,('/doc' , 'documentation') ,('/sources' , 'sources') ,('/queries' , 'queries') ,('/top' , 'top %s unarchived' % cfg.get('app', 'items_per_page')) ,('/all' , 'all') ) app = Flask(__name__) app.secret_key = cfg.get('app', 'secret_key') class SourceForm(Form): #name, address, source_type, is_public=True, attributes={} name = TextField('Name' , [Required()]) source_type = TextField('Type' , [Required()]) address = TextField('Address' , [Required()]) submit = SubmitField('Submit' , [Required()]) @app.context_processor
# # (c) 2012- by adam tauber, <*****@*****.**> if __name__ == '__main__': from sys import path from os.path import realpath, dirname path.append(realpath(dirname(realpath(__file__)) + '/../../')) from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Text, DateTime, Boolean, PickleType from sqlalchemy.orm import relationship, scoped_session, sessionmaker, backref from sqlalchemy.ext.declarative import declarative_base from datetime import datetime from pickle import dumps, loads from potion.common import cfg engine = create_engine(cfg.get('database', 'connection')) db_session = scoped_session( sessionmaker(autocommit=False, autoflush=False, bind=engine)) Base = declarative_base() Base.metadata.bind = engine Base.query = db_session.query_property() class User(Base): __tablename__ = 'users' user_id = Column(Integer, primary_key=True) name = Column(String(511)) password = Column(String(511)) is_active = Column(Boolean)
import urllib2 from potion.common import cfg if cfg.has_section('proxy') and cfg.has_option('proxy','proxy_type') and cfg.has_option('proxy','proxy_address'): proxy_type = cfg.get('proxy', 'proxy_type') proxy_address = cfg.get('proxy', 'proxy_address') import socks (proxy_host,proxy_port)=proxy_address.split(':') try: proxy_port=int(proxy_port) except ValueError: proxy_port=None print '[EE]proxy port must be int!' if not proxy_host or not proxy_port: #maybe mispelled config, exiting with code 'incorrect usage' print '[EE]Error in proxy address configuration(current value: %s)!' % proxy_address import sys sys.exit(2) socks.setdefaultproxy(eval('socks.PROXY_TYPE_'+proxy_type), proxy_host, int(proxy_port)) socks.wrapmodule(urllib2)
from flask import Flask, request, render_template, redirect, flash from sqlalchemy import not_ from potion.models import db_session, Item, Source, Query from potion.common import cfg from flask.ext.wtf import Form from wtforms import TextField, SubmitField from wtforms.validators import Required from potion.helpers import Pagination menu_items = (('/' , 'home') #,('/doc' , 'documentation') ,('/sources' , 'sources') ,('/queries' , 'queries') ,('/top' , 'top %s unarchived' % cfg.get('app', 'items_per_page')) ,('/saved' , 'top %s saved' % cfg.get('app', 'items_per_page')) ,('/all' , 'all') ) app = Flask(__name__) app.secret_key = cfg.get('app', 'secret_key') class SourceForm(Form): #name, address, source_type, is_public=True, attributes={} name = TextField('Name' , [Required()]) source_type = TextField('Type' , [Required()]) address = TextField('Address' , [Required()]) submit = SubmitField('Submit' , [Required()])
# # (c) 2012- by adam tauber, <*****@*****.**> if __name__ == '__main__': from sys import path from os.path import realpath, dirname path.append(realpath(dirname(realpath(__file__))+'/../../')) from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Text, DateTime, Boolean, PickleType from sqlalchemy.orm import relationship, scoped_session, sessionmaker, backref from sqlalchemy.ext.declarative import declarative_base from datetime import datetime from pickle import dumps, loads from potion.common import cfg engine = create_engine(cfg.get('database', 'connection')) db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) Base = declarative_base() Base.metadata.bind = engine Base.query = db_session.query_property() class User(Base): __tablename__ = 'users' user_id = Column(Integer, primary_key=True) name = Column(String(511)) password = Column(String(511)) is_active = Column(Boolean)
from flask import Flask from flask.ext.sqlalchemy import SQLAlchemy from potion.common import cfg app = Flask(__name__) app.secret_key = cfg.get('app', 'secret_key') app.config['SQLALCHEMY_DATABASE_URI'] = cfg.get('database', 'connection') db = SQLAlchemy(app)