def handle(self, args):
for newpath in settings.SCRAPER_PATHS:
sys.path.insert(0, newpath)
if args.sample:
bill = db.bills.find({'_id': args.sample})[0]
print bill_to_elasticsearch(bill)
else:
for abbr in args.abbrs:
for bill in db.bills.find({settings.LEVEL_FIELD: abbr.lower()},
timeout=False):
elasticsearch_push(bill)
def handle(self, args):
for newpath in settings.SCRAPER_PATHS:
sys.path.insert(0, newpath)
if args.sample:
bill = db.bills.find({'_id': args.sample})[0]
print bill_to_elasticsearch(bill)
else:
for abbr in args.abbrs:
for bill in db.bills.find({settings.LEVEL_FIELD: abbr.lower()},
timeout=False):
elasticsearch_push(bill)
def elasticsearch_push(bill):
if settings.ENABLE_ELASTICSEARCH_PUSH:
esdoc = bill_to_elasticsearch(bill)
elasticsearch.index(index='billy',
doc_type='bills',
id=bill['_id'],
doc=esdoc)
def elasticsearch_push(bill):
if settings.ENABLE_ELASTICSEARCH_PUSH:
esdoc = bill_to_elasticsearch(bill)
elasticsearch.index(index='billy', doc_type='bills', id=bill['_id'],
doc=esdoc)