Ejemplo n.º 1
0
def query_yahoo(index,numOfDays=3):
    """ query yahoo index content"""
    num_days = dt.now() - timedelta(days = numOfDays)

    filter = {'yahoo_market_news':0} if index == 'business' else {'yahoo_market_news':1}
    filter.update({'timestamp': {"$gt": num_days}})

    coll = default_cfg.MONGODB_COLLECTIONS['yahoofin']
    rets = []
    for i in client.scrapy[coll].find(filter).sort([('timestamp',DESCENDING)]):
        rets.append(dict(title=i['title'],source=i['source'],
                         date=timesince(i['timestamp']),url=i['url'],
                         oid=str(i['_id'])))
    return rets
Ejemplo n.º 2
0
def query_wantTimes(numOfDays=3):
    """ query wantTimes index content """
    num_days = dt.now() - timedelta(days=numOfDays)

    filter = {'timestamp': {"$gt": num_days}}
    
    coll = default_cfg.MONGODB_COLLECTIONS['wantTimes']
    # print coll
    # print client.scrapy[coll].count()
    rets = []
    for i in client.scrapy[coll].find(filter).sort([('timestamp',DESCENDING)]):
        rets.append(dict(title=i['title'],source=i['source'],
                         date=timesince(i['timestamp']),url=i['url'],
                         oid=str(i['_id'])))
    return rets
Ejemplo n.º 3
0
def query_items(items):
    """ """
    filter = {'url': {'$in': items}}
    cur = db.items.find(filter).sort([('timestamp',DESCENDING)])
    print cur.count()
    return [dict(title=i['title'],oid=str(i['_id']),date=timesince(i['timestamp'])) for i in cur]