예제 #1
0
파일: webapp.py 프로젝트: v-p-b/potion
def save(save_id=0):
    db_session.query(Item).filter(Item.item_id==save_id).update({Item.saved: True}, synchronize_session='fetch')
    db_session.commit()
    if save_id:
        return render_template('status.html', messages=['item(%s) saved' % save_id])
    flash('Successfully saved item')
    return redirect(request.referrer or '/')
예제 #2
0
파일: webapp.py 프로젝트: stef/potion
def archive():
    try:
        ids = map(int, request.form.get('ids', '').split(','))
    except:
        flash('Bad params')
        return redirect(request.referrer or '/')
    db_session.query(Item).filter(Item.item_id.in_(ids)).update({Item.archived: True}, synchronize_session='fetch')
    db_session.commit()
    flash('Successfully archived items: %d' % len(ids))
    return redirect(request.referrer or '/')
예제 #3
0
파일: cmd_cli.py 프로젝트: Fingel/potion
def add(source_name, item_name, content, url=None, attributes=None):
    if not attributes: attributes = {}
    source = db_session.query(Source).filter(Source.name==source_name).first() or Source(name=source_name, source_type='cmdline', address='/query/_%s' % source_name)
    item = Item(item_name, content, url=url, attributes=attributes)
    db_session.add(source)
    source.items.append(item)
    db_session.commit()
    return item
예제 #4
0
def archive(id=0):
    if request.method=='POST':
        try:
            ids = map(int, request.form.get('ids', '').split(','))
        except:
            flash('Bad params')
            return redirect(request.referrer or '/')
    elif id==0:
        flash('Nothing to archive')
        return redirect(request.referrer or '/')
    else:
        ids=[id]
    db_session.query(Item).filter(Item.item_id.in_(ids)).update({Item.archived: True}, synchronize_session='fetch')
    db_session.commit()
    if id:
        return render_template('status.html', messages=['item(%s) archived' % id])
    flash('Successfully archived items: %d' % len(ids))
    return redirect(request.referrer or '/')
예제 #5
0
파일: feed.py 프로젝트: stef/potion
def parseFeed(feed):
    counter = 0
    #modified = feed['modified'].timetuple() if feed.get('modified') else None
    f = None
    f = parse(fetchFeed(feed.address)
             ,etag      = feed.attributes.get('etag')
             ,modified  = feed.attributes.get('modified')
             )
    if not f:
        print '[EE] cannot parse %s - %s' % (feed.name, feed.address)
        return counter
    #print '[!] parsing %s - %s' % (feed.name, feed.url)
    try:
        feed.attributes['etag'] = f.etag
    except AttributeError:
        pass
    try:
        feed.attributes['modified'] = f.modified
    except AttributeError:
        pass
    d = feed.updated
    for item in reversed(f['entries']):
        try:
           u = urlSanitize(item['links'][0]['href'])
        except:
           u = ''
        # checking duplications
        if db_session.query(Item).filter(Item.source_id==feed.source_id).filter(Item.url==u).first():
            continue

        try:
            tmp_date = datetime(*item['updated_parsed'][:6])
        except:
            tmp_date = datetime.now()

        # title content updated
        try:
            c = unicode(''.join([x.value for x in item.content]))
        except:
            c = u'[EE] No content found, plz check the feed (%s) and fix me' % feed.name
            for key in ['media_text', 'summary', 'description', 'media:description']:
                if item.has_key(key):
                    c = unicode(item[key])
                    break

        t = unicode(item.get('title','[EE] Notitle'))

        # date as tmp_date?!
        feed.items.append(Item(t, c, url=u, attributes={'date':tmp_date}))
        db_session.commit()
        counter += 1
    feed.updated = d
    db_session.commit()
    #feed.save()
    return counter
예제 #6
0
def add(source_name, item_name, content, url=None, attributes=None):
    if not attributes: attributes = {}
    source = db_session.query(Source).filter(
        Source.name == source_name).first() or Source(
            name=source_name,
            source_type='cmdline',
            address='/query/_%s' % source_name)
    item = Item(item_name, content, url=url, attributes=attributes)
    db_session.add(source)
    source.items.append(item)
    db_session.commit()
    return item
예제 #7
0
def do_query(q_str):
    page_num = 1
    if(q_str.find('/')):
        try:
            page_num = int(q_str.split('/')[-1])
            q_str = ''.join(q_str.split('/')[:-1])
        except:
            pass

    #if(q_str.startswith('!')):
    #    q_str = q_str[1:]
    #    reverse = True

    rules = q_str.split(',')
    query = db_session.query(Item).filter(Item.source_id==Source.source_id)
    for rule in rules:
        if rule.find(':') != -1:
            item, value = rule.split(':', 1)
            if item.startswith('~'):
                query = query.filter(getattr(Item, item[1:]).contains(value))
            elif item.startswith('-'):
                query = query.filter(not_(getattr(Item, item[1:]).contains(value)))
            else:
                query = query.filter(getattr(Item, item) == value)
            continue
        if rule.startswith('_'):
            query = query.filter(Source.name == rule[1:])
            continue
    count = query.count()
    limit = int(cfg.get('app', 'items_per_page'))
    offset = limit*(page_num-1)
    items = query.limit(limit).offset(offset).all()
    #if reverse:
    #    items.reverse()

    pagination = Pagination(page_num, limit, count)
    return render_template('flat.html'
                          ,pagination   = pagination
                          ,items        = items
                          ,unarchiveds  = get_unarchived_ids(items)
                          ,menu_path    = '/query/%s' % q_str
                          )
예제 #8
0
파일: feed.py 프로젝트: misnyo/potion
def parseFeed(feed):
    counter = 0
    # modified = feed['modified'].timetuple() if feed.get('modified') else None
    f = None
    f = parse(fetchFeed(feed.address), etag=feed.attributes.get("etag"), modified=feed.attributes.get("modified"))
    if not f:
        print "[EE] cannot parse %s - %s" % (feed.name, feed.address)
        return counter
    # print '[!] parsing %s - %s' % (feed.name, feed.url)
    try:
        if feed.attributes["etag"] != f.etag:
            return
    except KeyError:
        pass

    try:
        feed.attributes["etag"] = f.etag
    except AttributeError:
        pass

    try:
        feed.attributes["modified"] = f.modified
    except AttributeError:
        pass

    d = feed.updated
    for item in reversed(f["entries"]):
        if "links" in item:
            original_url = unicode(item["links"][0]["href"])
        else:
            original_url = unicode(item["link"])

        # checking duplications
        if (
            db_session.query(Item)
            .filter(Item.source_id == feed.source_id)
            .filter(Item.original_url == original_url)
            .first()
        ):
            continue

        try:
            u = urlSanitize(original_url)
        except:
            u = ""

        try:
            tmp_date = datetime(*item["updated_parsed"][:6])
        except:
            tmp_date = datetime.now()

        # title content updated
        try:
            c = "".join([x.value for x in item.content])
        except:
            c = u"[EE] No content found, plz check the feed (%s) and fix me" % feed.name
            for key in ["media_text", "summary", "description", "media:description"]:
                if item.has_key(key):
                    c = item[key]
                    break

        # fixing malformed html
        if c:
            original = c
            c = ""
            try:
                phtml = etree.parse(StringIO(original), etree.HTMLParser())
                for node in phtml.iter("*"):
                    clean_description(node)
                for node in phtml.xpath("//body/*"):
                    c += etree.tostring(node)
            except:
                print u"[EE]description parsing error(%s - %s)" % (feed.name, u)
                c = original

        t = item.get("title", "[EE] Notitle")

        # date as tmp_date?!
        feed.items.append(Item(t, c, original_url, url=u, attributes={"date": tmp_date}))
        db_session.commit()
        counter += 1
    feed.updated = d
    db_session.commit()
    # feed.save()
    return counter
예제 #9
0
파일: feed.py 프로젝트: Fingel/potion
def parseFeed(feed):
    counter = 0
    #modified = feed['modified'].timetuple() if feed.get('modified') else None
    f = None
    f = parse(fetchFeed(feed.address)
             ,etag      = feed.attributes.get('etag')
             ,modified  = feed.attributes.get('modified')
             )
    if not f:
        print '[EE] cannot parse %s - %s' % (feed.name, feed.address)
        return counter
    #print '[!] parsing %s - %s' % (feed.name, feed.url)
    try:
        if feed.attributes['etag'] != f.etag:
            return
    except KeyError:
        pass

    try:
        feed.attributes['etag'] = f.etag
    except AttributeError:
        pass

    try:
        feed.attributes['modified'] = f.modified
    except AttributeError:
        pass

    d = feed.updated
    for item in reversed(f['entries']):
        if 'links' in item:
            original_url = unicode(item['links'][0]['href'])
        else:
            original_url = unicode(item['link'])

        # checking duplications
        if db_session.query(Item). \
                filter(Item.source_id==feed.source_id). \
                filter(Item.original_url==original_url).first():
            continue

        try:
           u = urlSanitize(original_url)
        except:
           u = ''

        try:
            tmp_date = datetime(*item['updated_parsed'][:6])
        except:
            tmp_date = datetime.now()

        # title content updated
        try:
            c = ''.join([x.value for x in item.content])
        except:
            c = u'[EE] No content found, plz check the feed (%s) and fix me' % feed.name
            for key in ['media_text', 'summary', 'description', 'media:description']:
                if item.has_key(key):
                    c = item[key]
                    break

#        #fixing malformed html
#        if c:
#            original = c
#            c = ''
#            try:
#                phtml = etree.parse(StringIO(original), etree.HTMLParser())
#                for node in phtml.iter('*'):
#                    clean_description(node)
#                for node in phtml.xpath('//body/*'):
#                    c += etree.tostring(node)
#            except:
#                print u'[EE]description parsing error(%s - %s)' % (feed.name, u)
#                c = original

        t = item.get('title','[EE] Notitle')

        # date as tmp_date?!
        feed.items.append(Item(t, c, original_url, url=u, attributes={'date':tmp_date}))
        db_session.commit()
        counter += 1
    feed.updated = d
    db_session.commit()
    #feed.save()
    return counter
예제 #10
0
def parseFeed(feed):
    counter = 0
    #modified = feed['modified'].timetuple() if feed.get('modified') else None
    f = None
    f = parse(fetchFeed(feed.address),
              etag=feed.attributes.get('etag'),
              modified=feed.attributes.get('modified'))
    if not f:
        print '[EE] cannot parse %s - %s' % (feed.name, feed.address)
        return counter
    #print '[!] parsing %s - %s' % (feed.name, feed.url)
    try:
        if feed.attributes['etag'] != f.etag:
            return
    except KeyError:
        pass

    try:
        feed.attributes['etag'] = f.etag
    except AttributeError:
        pass

    try:
        feed.attributes['modified'] = f.modified
    except AttributeError:
        pass

    d = feed.updated
    for item in reversed(f['entries']):
        if 'links' in item:
            original_url = unicode(item['links'][0]['href'])
        else:
            original_url = unicode(item['link'])

        # checking duplications
        if db_session.query(Item). \
                filter(Item.source_id==feed.source_id). \
                filter(Item.original_url==original_url).first():
            continue

        try:
            u = urlSanitize(original_url)
        except:
            u = ''

        try:
            tmp_date = datetime(*item['updated_parsed'][:6])
        except:
            tmp_date = datetime.now()

        # title content updated
        try:
            c = ''.join([x.value for x in item.content])
        except:
            c = u'[EE] No content found, plz check the feed (%s) and fix me' % feed.name
            for key in [
                    'media_text', 'summary', 'description', 'media:description'
            ]:
                if item.has_key(key):
                    c = item[key]
                    break

#        #fixing malformed html
#        if c:
#            original = c
#            c = ''
#            try:
#                phtml = etree.parse(StringIO(original), etree.HTMLParser())
#                for node in phtml.iter('*'):
#                    clean_description(node)
#                for node in phtml.xpath('//body/*'):
#                    c += etree.tostring(node)
#            except:
#                print u'[EE]description parsing error(%s - %s)' % (feed.name, u)
#                c = original

        t = item.get('title', '[EE] Notitle')

        # date as tmp_date?!
        feed.items.append(
            Item(t, c, original_url, url=u, attributes={'date': tmp_date}))
        db_session.commit()
        counter += 1
    feed.updated = d
    db_session.commit()
    #feed.save()
    return counter