def _do_imports(abbrev, args): # do imports here so that scrape doesn't depend on mongo from billy.importers.metadata import import_metadata from billy.importers.bills import import_bills from billy.importers.legislators import import_legislators from billy.importers.committees import import_committees from billy.importers.events import import_events from billy.importers.speeches import import_speeches # always import metadata and districts import_metadata(abbrev) report = {} if 'legislators' in args.types: report['legislators'] = \ import_legislators(abbrev, settings.BILLY_DATA_DIR) if 'bills' in args.types: report['bills'] = import_bills(abbrev, settings.BILLY_DATA_DIR) if 'committees' in args.types: report['committees'] = \ import_committees(abbrev, settings.BILLY_DATA_DIR) if 'events' in args.types or 'speeches' in args.types: report['events'] = import_events(abbrev, settings.BILLY_DATA_DIR) if 'speeches' in args.types: report['speeches'] = import_speeches(abbrev, settings.BILLY_DATA_DIR) return report
def _do_imports(abbrev, args): # do imports here so that scrape doesn't depend on mongo from billy.importers.metadata import import_metadata from billy.importers.bills import import_bills from billy.importers.legislators import import_legislators from billy.importers.committees import import_committees from billy.importers.events import import_events from billy.importers.speeches import import_speeches # always import metadata and districts import_metadata(abbrev) dist_filename = os.path.join(settings.BILLY_MANUAL_DATA_DIR, 'districts', '%s.csv' % abbrev) if os.path.exists(dist_filename): db.districts.remove({'abbr': abbrev}) dist_csv = unicodecsv.DictReader(open(dist_filename)) for dist in dist_csv: dist['_id'] = '%(abbr)s-%(chamber)s-%(name)s' % dist dist['boundary_id'] = dist['boundary_id'] % dist dist['num_seats'] = int(dist['num_seats']) _log.debug(dist) db.districts.save(dist, safe=True) else: logging.getLogger('billy').warning("%s not found, continuing without " "districts" % dist_filename) report = {} if 'legislators' in args.types: report['legislators'] = import_legislators( abbrev, settings.BILLY_DATA_DIR ) if 'bills' in args.types: report['bills'] = import_bills(abbrev, settings.BILLY_DATA_DIR) if 'committees' in args.types: report['committees'] = import_committees( abbrev, settings.BILLY_DATA_DIR ) if 'events' in args.types or 'speeches' in args.types: report['events'] = import_events(abbrev, settings.BILLY_DATA_DIR) if 'speeches' in args.types: report['speeches'] = import_speeches(abbrev, settings.BILLY_DATA_DIR) return report
def _do_imports(abbrev, args): # do imports here so that scrape doesn't depend on mongo from billy.importers.metadata import import_metadata from billy.importers.bills import import_bills from billy.importers.legislators import import_legislators from billy.importers.committees import import_committees from billy.importers.events import import_events from billy.importers.speeches import import_speeches # always import metadata and districts import_metadata(abbrev) dist_filename = os.path.join(settings.BILLY_MANUAL_DATA_DIR, 'districts', '%s.csv' % abbrev) if os.path.exists(dist_filename): db.districts.remove({'abbr': abbrev}) dist_csv = unicodecsv.DictReader(open(dist_filename)) for dist in dist_csv: dist['_id'] = '%(abbr)s-%(chamber)s-%(name)s' % dist dist['boundary_id'] = dist['boundary_id'] % dist dist['num_seats'] = int(dist['num_seats']) db.districts.save(dist, safe=True) else: logging.getLogger('billy').warning("%s not found, continuing without " "districts" % dist_filename) report = {} if 'legislators' in args.types: report['legislators'] = \ import_legislators(abbrev, settings.BILLY_DATA_DIR) if 'bills' in args.types: report['bills'] = import_bills(abbrev, settings.BILLY_DATA_DIR) if 'committees' in args.types: report['committees'] = \ import_committees(abbrev, settings.BILLY_DATA_DIR) if 'events' in args.types or 'speeches' in args.types: report['events'] = import_events(abbrev, settings.BILLY_DATA_DIR) if 'speeches' in args.types: report['speeches'] = import_speeches(abbrev, settings.BILLY_DATA_DIR) return report