def import_all(*abbrs): for state in openstates.all_metadata(): if abbrs and state["abbreviation"] not in abbrs: continue abbr = state["abbreviation"] # if abbr in ('co', 'ar', 'ct', 'al', 'dc', 'id'): # continue meta = openstates.state_metadata(abbr) for chamber in meta["chambers"]: latest_term = sorted(meta["terms"], key=itemgetter("start_year")) term = latest_term.pop() spec = dict(term=term["name"], abbr=abbr, chamber=chamber) if mongo.reports.find_one(spec): logging.debug("Skipping %r" % spec) continue try: calc = ScoreCalculator(abbr, chamber, meta, term=term["name"]) report = calc.import_scores(meta) except DataQualityError as exc: logging.exception(exc) logging.error("No party data: skipping %r" % ([abbr, chamber, term],)) # import pdb; pdb.set_trace() except TooFewBillsError as exc: logging.exception(exc) logging.error("Too few bills found: skipping %r" % ([abbr, chamber, term],)) # import pdb; pdb.set_trace() except rpy2.rinterface.RRuntimeError as exc: logging.exception(exc) logging.error("R error: skipping %r" % ([abbr, chamber, term],))
def import_all(*abbrs): for state in openstates.all_metadata(): if abbrs and state['abbreviation'] not in abbrs: continue abbr = state['abbreviation'] if abbr in ('co', 'ar', 'ct', 'al', 'dc', 'id'): continue meta = openstates.state_metadata(abbr) for chamber in meta['chambers']: latest_term = sorted(meta['terms'], key=itemgetter('start_year')) term = latest_term.pop() spec = dict( term=term['name'], abbr=abbr, chamber=chamber) if mongo.reports.find_one(spec): logging.debug('Skipping %r' % spec) continue try: calc = ScoreCalculator(abbr, chamber, meta, term=term['name']) report = calc.import_scores(meta) except DataQualityError as exc: logging.exception(exc) logging.error('No party data: skipping %r' % ([abbr, chamber, term],)) except TooFewBillsError as exc: logging.exception(exc) logging.error('Too few bills found: skipping %r' % ([abbr, chamber, term],)) except rpy2.rinterface.RRuntimeError as exc: logging.exception(exc) logging.error('R error: skipping %r' % ([abbr, chamber, term],))
#!/usr/bin/env python from sunlight import openstates import json states = openstates.all_metadata() ret = {} for state in states: abbr = state['abbreviation'] ret[abbr] = { "upper": [], "lower": [] } legislators = openstates.legislators(state=abbr) for leg in legislators: if "chamber" not in leg: continue # Dewhurst. chamber = leg['chamber'] party = leg['party'] ret[abbr][chamber].append(party[0]) open("openstates.json", 'w').write(json.dumps(ret))
# FUNCTION TO PARSE STATE METADATA def parse_state_metadata(state_metadata): name = state_metadata['name'] abbreviation = state_metadata['abbreviation'] if 'lower' in state_metadata['chambers']: lower_chamber_name = state_metadata['chambers']['lower']['name'] lower_chamber_title = state_metadata['chambers']['lower']['title'] else: lower_chamber_name = None lower_chamber_title = None upper_chamber_name = state_metadata['chambers']['upper']['name'] upper_chamber_title = state_metadata['chambers']['upper']['title'] feature_flags = ', '.join(state_metadata['feature_flags']) return((name, abbreviation, lower_chamber_name, lower_chamber_title, upper_chamber_name, upper_chamber_name, feature_flags)) # GRAB THE DATA FROM SUNLIGHT API state_metadata = openstates.all_metadata() # PARSE SUNLIGHT DATA AND WRITE TO POSTGRES temp_state_metadata = [] for state in state_metadata: temp_state_metadata.append(parse_state_metadata(state)) args_str = ','.join(cur.mogrify("(%s,%s,%s,%s,%s,%s,%s)", x) for x in temp_state_metadata) cur.execute("INSERT INTO state_metadata VALUES " + args_str) conn.commit()