def status():
init_django()
from opencivicdata.legislative.models import Bill
states = sorted(CONVERSION_FUNCTIONS.keys())
click.secho("state | bills | missing | errors ", fg="white")
click.secho("===================================", fg="white")
for state in states:
all_bills = Bill.objects.filter(legislative_session__jurisdiction_id=abbr_to_jid(state))
missing_search = all_bills.filter(searchable__isnull=True).count()
errors = all_bills.filter(searchable__is_error=True).count()
all_bills = all_bills.count()
errcolor = mscolor = "green"
if missing_search > 0:
missing_search = math.ceil(missing_search / all_bills * 100)
mscolor = "yellow"
if missing_search > 1:
mscolor = "red"
if errors > 0:
errcolor = "yellow"
errors = math.ceil(errors / all_bills * 100)
if errors > 5:
errcolor = "red"
click.echo(
f"{state:5} | {all_bills:7} | "
+ click.style(f"{missing_search:6}%", fg=mscolor)
+ " | "
+ click.style(f"{errors:6}%", fg=errcolor)
)
def reindex_state(state):
init_django()
from openstates.data.models import SearchableBill
ids = list(
SearchableBill.objects.filter(
bill__legislative_session__jurisdiction_id=abbr_to_jid(
state)).values_list("id", flat=True))
print(f"reindexing {len(ids)} bills for state")
reindex(ids)
def _resample(state, n=50):
"""
Grab new versions for a state from the database.
"""
init_django()
from opencivicdata.legislative.models import BillVersion
versions = BillVersion.objects.filter(
bill__legislative_session__jurisdiction_id=abbr_to_jid(state)
).order_by("?")[:n]
count = 0
fieldnames = [
"id",
"session",
"identifier",
"title",
"jurisdiction_id",
"media_type",
"url",
"note",
]
with open(f"raw/{state}.csv", "w") as outf:
out = csv.DictWriter(outf, fieldnames=fieldnames)
out.writeheader()
for v in versions:
for link in v.links.all():
out.writerow(
{
"id": v.id,
"session": v.bill.legislative_session.identifier,
"jurisdiction_id": v.bill.legislative_session.jurisdiction_id,
"identifier": v.bill.identifier,
"title": v.bill.title,
"url": link.url,
"media_type": link.media_type,
"note": v.note,
}
)
count += 1
click.secho(f"wrote new sample csv with {count} records")
def update(state, n, clear_errors, checkpoint):
init_django()
from opencivicdata.legislative.models import Bill, SearchableBill
# print status within checkpoints
status_num = checkpoint / 5
if state == "all":
all_bills = Bill.objects.all()
else:
all_bills = Bill.objects.filter(legislative_session__jurisdiction_id=abbr_to_jid(state))
if clear_errors:
if state == "all":
print("--clear-errors only works with specific states, not all")
return
errs = SearchableBill.objects.filter(bill__in=all_bills, is_error=True)
print(f"clearing {len(errs)} errors")
errs.delete()
missing_search = all_bills.filter(searchable__isnull=True)
if state == "all":
MAX_UPDATE = 500
aggregates = missing_search.values("legislative_session__jurisdiction__name").annotate(
count=Count("id")
)
bail = False
for agg in aggregates:
state_name = agg["legislative_session__jurisdiction__name"]
if agg["count"] > MAX_UPDATE:
click.secho(f"Too many bills to update for {state_name}: {agg['count']}", fg="red")
bail = True
if bail:
sys.exit(1)
print(f"{len(missing_search)} missing, updating")
else:
print(f"{state}: {len(all_bills)} bills, {len(missing_search)} without search results")
if n:
missing_search = missing_search[: int(n)]
else:
n = len(missing_search)
ids_to_update = []
updated_count = 0
# going to manage our own transactions here so we can save in chunks
transaction.set_autocommit(False)
for b in missing_search:
ids_to_update.append(update_bill(b))
updated_count += 1
if updated_count % status_num == 0:
print(f"{state}: updated {updated_count} out of {n}")
if updated_count % checkpoint == 0:
reindex(ids_to_update)
transaction.commit()
ids_to_update = []
# be sure to reindex final set
reindex(ids_to_update)
transaction.commit()
transaction.set_autocommit(True)
