def status() -> None:
init_django()
from openstates.data.models import Bill
states = sorted(CONVERSION_FUNCTIONS.keys())
click.secho("state | bills | missing | errors ", fg="white")
click.secho("===================================", fg="white")
for state in states:
all_bills = Bill.objects.filter(
legislative_session__jurisdiction_id=abbr_to_jid(state)
)
missing_search = all_bills.filter(searchable__isnull=True).count()
errors = all_bills.filter(searchable__is_error=True).count()
all_bills = all_bills.count()
errcolor = mscolor = "green"
if missing_search > 0:
missing_search = math.ceil(missing_search / all_bills * 100)
mscolor = "yellow"
if missing_search > 1:
mscolor = "red"
if errors > 0:
errcolor = "yellow"
errors = math.ceil(errors / all_bills * 100)
if errors > 5:
errcolor = "red"
click.echo(
f"{state:5} | {all_bills:7} | "
+ click.style(f"{missing_search:6}%", fg=mscolor)
+ " | "
+ click.style(f"{errors:6}%", fg=errcolor)
)
def reindex_state(state: str) -> None:
init_django()
from openstates.data.models import SearchableBill
ids = list(
SearchableBill.objects.filter(
bill__legislative_session__jurisdiction_id=abbr_to_jid(
state)).values_list("id", flat=True))
print(f"reindexing {len(ids)} bills for state")
reindex(ids)
def to_database(abbreviations, purge, safe):
"""
Sync YAML files to DB.
"""
init_django()
create_parties()
if not abbreviations:
abbreviations = get_all_abbreviations()
for abbr in abbreviations:
click.secho("==== {} ====".format(abbr), bold=True)
directory = get_data_dir(abbr)
jurisdiction_id = get_jurisdiction_id(abbr)
municipalities = load_municipalities(abbr)
with transaction.atomic():
create_municipalities(municipalities)
person_files = (
glob.glob(os.path.join(directory, "legislature/*.yml")) +
glob.glob(os.path.join(directory, "executive/*.yml")) +
glob.glob(os.path.join(directory, "municipalities/*.yml")) +
glob.glob(os.path.join(directory, "retired/*.yml")))
committee_files = glob.glob(
os.path.join(directory, "organizations/*.yml"))
if safe:
click.secho("running in safe mode, no changes will be made",
fg="magenta")
try:
with transaction.atomic():
load_directory(person_files,
"person",
jurisdiction_id,
purge=purge)
load_directory(committee_files,
"organization",
jurisdiction_id,
purge=purge)
if safe:
click.secho("ran in safe mode, no changes were made",
fg="magenta")
raise CancelTransaction()
except CancelTransaction:
sys.exit(1)
def _resample(state: str, n: int = 50) -> None:
"""
Grab new versions for a state from the database.
"""
init_django()
from openstates.data.models import BillVersion
versions = BillVersion.objects.filter(
bill__legislative_session__jurisdiction_id=abbr_to_jid(state)
).order_by("?")[:n]
count = 0
fieldnames = [
"id",
"session",
"identifier",
"title",
"jurisdiction_id",
"media_type",
"url",
"note",
]
with open(get_raw_dir() / f"{state}.csv", "w") as outf:
out = csv.DictWriter(outf, fieldnames=fieldnames)
out.writeheader()
for v in versions:
for link in v.links.all():
out.writerow(
{
"id": v.id,
"session": v.bill.legislative_session.identifier,
"jurisdiction_id": v.bill.legislative_session.jurisdiction_id,
"identifier": v.bill.identifier,
"title": v.bill.title,
"url": link.url,
"media_type": link.media_type,
"note": v.note,
}
)
count += 1
click.secho(f"wrote new sample csv with {count} records")
def to_database(abbreviations: list[str], purge: bool, safe: bool) -> None:
"""
Sync YAML files to DB.
"""
init_django()
create_parties()
if not abbreviations:
abbreviations = get_all_abbreviations()
for abbr in abbreviations:
click.secho("==== {} ====".format(abbr), bold=True)
directory = get_data_path(abbr)
municipalities = load_municipalities(abbr)
with transaction.atomic():
create_municipalities(municipalities)
person_files = list(
itertools.chain(
directory.glob("legislature/*.yml"),
directory.glob("executive/*.yml"),
directory.glob("municipalities/*.yml"),
directory.glob("retired/*.yml"),
)
)
if safe:
click.secho("running in safe mode, no changes will be made", fg="magenta")
try:
with transaction.atomic():
load_directory_to_database(person_files, purge=purge)
if safe:
click.secho("ran in safe mode, no changes were made", fg="magenta")
raise CancelTransaction()
except CancelTransaction:
sys.exit(1)
def update(
state: str, n: int, clear_errors: bool, checkpoint: int, session: str = None
) -> None:
init_django()
from openstates.data.models import Bill, SearchableBill
# print status within checkpoints
status_num = checkpoint / 5
if state == "all":
all_bills = Bill.objects.all()
elif session:
all_bills = Bill.objects.filter(
legislative_session__jurisdiction_id=abbr_to_jid(state),
legislative_session__identifier=session,
)
else:
all_bills = Bill.objects.filter(
legislative_session__jurisdiction_id=abbr_to_jid(state)
)
if clear_errors:
if state == "all":
print("--clear-errors only works with specific states, not all")
return
errs = SearchableBill.objects.filter(bill__in=all_bills, is_error=True)
print(f"clearing {len(errs)} errors")
errs.delete()
missing_search = all_bills.filter(searchable__isnull=True)
if state == "all":
MAX_UPDATE = 1000
aggregates = missing_search.values(
"legislative_session__jurisdiction__name"
).annotate(count=Count("id"))
for agg in aggregates:
state_name = agg["legislative_session__jurisdiction__name"]
if agg["count"] > MAX_UPDATE:
click.secho(
f"Too many bills to update for {state_name}: {agg['count']}, skipping",
fg="red",
)
missing_search = missing_search.exclude(
legislative_session__jurisdiction__name=state_name
)
print(f"{len(missing_search)} missing, updating")
else:
print(
f"{state}: {len(all_bills)} bills, {len(missing_search)} without search results"
)
if n:
missing_search = missing_search[: int(n)]
else:
n = len(missing_search)
ids_to_update = []
updated_count = 0
# going to manage our own transactions here so we can save in chunks
transaction.set_autocommit(False)
for b in missing_search:
ids_to_update.append(update_bill(b))
updated_count += 1
if updated_count % status_num == 0:
print(f"{state}: updated {updated_count} out of {n}")
if updated_count % checkpoint == 0:
reindex(ids_to_update)
transaction.commit()
ids_to_update = []
# be sure to reindex final set
reindex(ids_to_update)
transaction.commit()
transaction.set_autocommit(True)
jurisdiction_id = get_jurisdiction_id(abbr)
voters, sponsorships = get_unmatched(jurisdiction_id)
update_objects(jurisdiction_id, voters, "vote", dry)
update_objects(jurisdiction_id, sponsorships, "sponsorship", dry)
@click.command()
@click.argument("abbreviations", nargs=-1)
@click.option("--dump/--no-dump")
@click.option("--match/--no-match")
@click.option("--dry/--no-dry", default=True)
def process_unmatched(abbreviations, dump, match, dry):
if not abbreviations:
abbreviations = get_all_abbreviations()
for abbr in abbreviations:
if match:
if dry:
click.secho("dry run, nothing will be saved", fg="blue")
try:
check_historical_matches(abbr, dry=dry)
except AbortTransaction as e:
click.secho(f"{e}\ntransaction aborted!", fg="red")
if dump:
archive_leg_to_csv(abbr)
if __name__ == "__main__":
init_django()
process_unmatched()