def query_yahoo(index,numOfDays=3):
""" query yahoo index content"""
num_days = dt.now() - timedelta(days = numOfDays)
filter = {'yahoo_market_news':0} if index == 'business' else {'yahoo_market_news':1}
filter.update({'timestamp': {"$gt": num_days}})
coll = default_cfg.MONGODB_COLLECTIONS['yahoofin']
rets = []
for i in client.scrapy[coll].find(filter).sort([('timestamp',DESCENDING)]):
rets.append(dict(title=i['title'],source=i['source'],
date=timesince(i['timestamp']),url=i['url'],
oid=str(i['_id'])))
return rets
def query_wantTimes(numOfDays=3):
""" query wantTimes index content """
num_days = dt.now() - timedelta(days=numOfDays)
filter = {'timestamp': {"$gt": num_days}}
coll = default_cfg.MONGODB_COLLECTIONS['wantTimes']
# print coll
# print client.scrapy[coll].count()
rets = []
for i in client.scrapy[coll].find(filter).sort([('timestamp',DESCENDING)]):
rets.append(dict(title=i['title'],source=i['source'],
date=timesince(i['timestamp']),url=i['url'],
oid=str(i['_id'])))
return rets
def query_items(items):
""" """
filter = {'url': {'$in': items}}
cur = db.items.find(filter).sort([('timestamp',DESCENDING)])
print cur.count()
return [dict(title=i['title'],oid=str(i['_id']),date=timesince(i['timestamp'])) for i in cur]