def query_yahoo(index,numOfDays=3): """ query yahoo index content""" num_days = dt.now() - timedelta(days = numOfDays) filter = {'yahoo_market_news':0} if index == 'business' else {'yahoo_market_news':1} filter.update({'timestamp': {"$gt": num_days}}) coll = default_cfg.MONGODB_COLLECTIONS['yahoofin'] rets = [] for i in client.scrapy[coll].find(filter).sort([('timestamp',DESCENDING)]): rets.append(dict(title=i['title'],source=i['source'], date=timesince(i['timestamp']),url=i['url'], oid=str(i['_id']))) return rets
def query_wantTimes(numOfDays=3): """ query wantTimes index content """ num_days = dt.now() - timedelta(days=numOfDays) filter = {'timestamp': {"$gt": num_days}} coll = default_cfg.MONGODB_COLLECTIONS['wantTimes'] # print coll # print client.scrapy[coll].count() rets = [] for i in client.scrapy[coll].find(filter).sort([('timestamp',DESCENDING)]): rets.append(dict(title=i['title'],source=i['source'], date=timesince(i['timestamp']),url=i['url'], oid=str(i['_id']))) return rets
def query_items(items): """ """ filter = {'url': {'$in': items}} cur = db.items.find(filter).sort([('timestamp',DESCENDING)]) print cur.count() return [dict(title=i['title'],oid=str(i['_id']),date=timesince(i['timestamp'])) for i in cur]