def _do_imports(abbrev, args):
# do imports here so that scrape doesn't depend on mongo
from billy.importers.metadata import import_metadata
from billy.importers.bills import import_bills
from billy.importers.legislators import import_legislators
from billy.importers.committees import import_committees
from billy.importers.events import import_events
from billy.importers.speeches import import_speeches
# always import metadata and districts
import_metadata(abbrev)
report = {}
if 'legislators' in args.types:
report['legislators'] = \
import_legislators(abbrev, settings.BILLY_DATA_DIR)
if 'bills' in args.types:
report['bills'] = import_bills(abbrev, settings.BILLY_DATA_DIR)
if 'committees' in args.types:
report['committees'] = \
import_committees(abbrev, settings.BILLY_DATA_DIR)
if 'events' in args.types or 'speeches' in args.types:
report['events'] = import_events(abbrev, settings.BILLY_DATA_DIR)
if 'speeches' in args.types:
report['speeches'] = import_speeches(abbrev, settings.BILLY_DATA_DIR)
return report
def _do_imports(abbrev, args):
# do imports here so that scrape doesn't depend on mongo
from billy.importers.metadata import import_metadata
from billy.importers.bills import import_bills
from billy.importers.legislators import import_legislators
from billy.importers.committees import import_committees
from billy.importers.events import import_events
from billy.importers.speeches import import_speeches
# always import metadata and districts
import_metadata(abbrev)
dist_filename = os.path.join(settings.BILLY_MANUAL_DATA_DIR, 'districts',
'%s.csv' % abbrev)
if os.path.exists(dist_filename):
db.districts.remove({'abbr': abbrev})
dist_csv = unicodecsv.DictReader(open(dist_filename))
for dist in dist_csv:
dist['_id'] = '%(abbr)s-%(chamber)s-%(name)s' % dist
dist['boundary_id'] = dist['boundary_id'] % dist
dist['num_seats'] = int(dist['num_seats'])
_log.debug(dist)
db.districts.save(dist, safe=True)
else:
logging.getLogger('billy').warning("%s not found, continuing without "
"districts" % dist_filename)
report = {}
if 'legislators' in args.types:
report['legislators'] = import_legislators(
abbrev,
settings.BILLY_DATA_DIR
)
if 'bills' in args.types:
report['bills'] = import_bills(abbrev, settings.BILLY_DATA_DIR)
if 'committees' in args.types:
report['committees'] = import_committees(
abbrev,
settings.BILLY_DATA_DIR
)
if 'events' in args.types or 'speeches' in args.types:
report['events'] = import_events(abbrev, settings.BILLY_DATA_DIR)
if 'speeches' in args.types:
report['speeches'] = import_speeches(abbrev, settings.BILLY_DATA_DIR)
return report
def _do_imports(abbrev, args):
# do imports here so that scrape doesn't depend on mongo
from billy.importers.metadata import import_metadata
from billy.importers.bills import import_bills
from billy.importers.legislators import import_legislators
from billy.importers.committees import import_committees
from billy.importers.events import import_events
from billy.importers.speeches import import_speeches
# always import metadata and districts
import_metadata(abbrev)
dist_filename = os.path.join(settings.BILLY_MANUAL_DATA_DIR, 'districts',
'%s.csv' % abbrev)
if os.path.exists(dist_filename):
db.districts.remove({'abbr': abbrev})
dist_csv = unicodecsv.DictReader(open(dist_filename))
for dist in dist_csv:
dist['_id'] = '%(abbr)s-%(chamber)s-%(name)s' % dist
dist['boundary_id'] = dist['boundary_id'] % dist
dist['num_seats'] = int(dist['num_seats'])
db.districts.save(dist, safe=True)
else:
logging.getLogger('billy').warning("%s not found, continuing without "
"districts" % dist_filename)
report = {}
if 'legislators' in args.types:
report['legislators'] = \
import_legislators(abbrev, settings.BILLY_DATA_DIR)
if 'bills' in args.types:
report['bills'] = import_bills(abbrev, settings.BILLY_DATA_DIR)
if 'committees' in args.types:
report['committees'] = \
import_committees(abbrev, settings.BILLY_DATA_DIR)
if 'events' in args.types or 'speeches' in args.types:
report['events'] = import_events(abbrev, settings.BILLY_DATA_DIR)
if 'speeches' in args.types:
report['speeches'] = import_speeches(abbrev, settings.BILLY_DATA_DIR)
return report
