コード例 #1
0
def status() -> None:
    init_django()
    from openstates.data.models import Bill

    states = sorted(CONVERSION_FUNCTIONS.keys())
    click.secho("state |  bills  | missing | errors ", fg="white")
    click.secho("===================================", fg="white")
    for state in states:
        all_bills = Bill.objects.filter(
            legislative_session__jurisdiction_id=abbr_to_jid(state)
        )
        missing_search = all_bills.filter(searchable__isnull=True).count()
        errors = all_bills.filter(searchable__is_error=True).count()
        all_bills = all_bills.count()

        errcolor = mscolor = "green"
        if missing_search > 0:
            missing_search = math.ceil(missing_search / all_bills * 100)
            mscolor = "yellow"
        if missing_search > 1:
            mscolor = "red"
        if errors > 0:
            errcolor = "yellow"
            errors = math.ceil(errors / all_bills * 100)
        if errors > 5:
            errcolor = "red"

        click.echo(
            f"{state:5} | {all_bills:7} | "
            + click.style(f"{missing_search:6}%", fg=mscolor)
            + " | "
            + click.style(f"{errors:6}%", fg=errcolor)
        )
コード例 #2
0
def reindex_state(state: str) -> None:
    init_django()
    from openstates.data.models import SearchableBill

    ids = list(
        SearchableBill.objects.filter(
            bill__legislative_session__jurisdiction_id=abbr_to_jid(
                state)).values_list("id", flat=True))
    print(f"reindexing {len(ids)} bills for state")
    reindex(ids)
コード例 #3
0
def to_database(abbreviations, purge, safe):
    """
    Sync YAML files to DB.
    """
    init_django()

    create_parties()

    if not abbreviations:
        abbreviations = get_all_abbreviations()

    for abbr in abbreviations:
        click.secho("==== {} ====".format(abbr), bold=True)
        directory = get_data_dir(abbr)
        jurisdiction_id = get_jurisdiction_id(abbr)
        municipalities = load_municipalities(abbr)

        with transaction.atomic():
            create_municipalities(municipalities)

        person_files = (
            glob.glob(os.path.join(directory, "legislature/*.yml")) +
            glob.glob(os.path.join(directory, "executive/*.yml")) +
            glob.glob(os.path.join(directory, "municipalities/*.yml")) +
            glob.glob(os.path.join(directory, "retired/*.yml")))
        committee_files = glob.glob(
            os.path.join(directory, "organizations/*.yml"))

        if safe:
            click.secho("running in safe mode, no changes will be made",
                        fg="magenta")

        try:
            with transaction.atomic():
                load_directory(person_files,
                               "person",
                               jurisdiction_id,
                               purge=purge)
                load_directory(committee_files,
                               "organization",
                               jurisdiction_id,
                               purge=purge)
                if safe:
                    click.secho("ran in safe mode, no changes were made",
                                fg="magenta")
                    raise CancelTransaction()
        except CancelTransaction:
            sys.exit(1)
コード例 #4
0
def _resample(state: str, n: int = 50) -> None:
    """
    Grab new versions for a state from the database.
    """
    init_django()
    from openstates.data.models import BillVersion

    versions = BillVersion.objects.filter(
        bill__legislative_session__jurisdiction_id=abbr_to_jid(state)
    ).order_by("?")[:n]

    count = 0
    fieldnames = [
        "id",
        "session",
        "identifier",
        "title",
        "jurisdiction_id",
        "media_type",
        "url",
        "note",
    ]

    with open(get_raw_dir() / f"{state}.csv", "w") as outf:
        out = csv.DictWriter(outf, fieldnames=fieldnames)
        out.writeheader()
        for v in versions:
            for link in v.links.all():
                out.writerow(
                    {
                        "id": v.id,
                        "session": v.bill.legislative_session.identifier,
                        "jurisdiction_id": v.bill.legislative_session.jurisdiction_id,
                        "identifier": v.bill.identifier,
                        "title": v.bill.title,
                        "url": link.url,
                        "media_type": link.media_type,
                        "note": v.note,
                    }
                )
                count += 1
    click.secho(f"wrote new sample csv with {count} records")
コード例 #5
0
ファイル: people.py プロジェクト: darshan-py7781/people
def to_database(abbreviations: list[str], purge: bool, safe: bool) -> None:
    """
    Sync YAML files to DB.
    """
    init_django()

    create_parties()

    if not abbreviations:
        abbreviations = get_all_abbreviations()

    for abbr in abbreviations:
        click.secho("==== {} ====".format(abbr), bold=True)
        directory = get_data_path(abbr)
        municipalities = load_municipalities(abbr)

        with transaction.atomic():
            create_municipalities(municipalities)

        person_files = list(
            itertools.chain(
                directory.glob("legislature/*.yml"),
                directory.glob("executive/*.yml"),
                directory.glob("municipalities/*.yml"),
                directory.glob("retired/*.yml"),
            )
        )

        if safe:
            click.secho("running in safe mode, no changes will be made", fg="magenta")

        try:
            with transaction.atomic():
                load_directory_to_database(person_files, purge=purge)
                if safe:
                    click.secho("ran in safe mode, no changes were made", fg="magenta")
                    raise CancelTransaction()
        except CancelTransaction:
            sys.exit(1)
コード例 #6
0
def update(
    state: str, n: int, clear_errors: bool, checkpoint: int, session: str = None
) -> None:
    init_django()
    from openstates.data.models import Bill, SearchableBill

    # print status within checkpoints
    status_num = checkpoint / 5

    if state == "all":
        all_bills = Bill.objects.all()
    elif session:
        all_bills = Bill.objects.filter(
            legislative_session__jurisdiction_id=abbr_to_jid(state),
            legislative_session__identifier=session,
        )
    else:
        all_bills = Bill.objects.filter(
            legislative_session__jurisdiction_id=abbr_to_jid(state)
        )

    if clear_errors:
        if state == "all":
            print("--clear-errors only works with specific states, not all")
            return
        errs = SearchableBill.objects.filter(bill__in=all_bills, is_error=True)
        print(f"clearing {len(errs)} errors")
        errs.delete()

    missing_search = all_bills.filter(searchable__isnull=True)
    if state == "all":
        MAX_UPDATE = 1000
        aggregates = missing_search.values(
            "legislative_session__jurisdiction__name"
        ).annotate(count=Count("id"))
        for agg in aggregates:
            state_name = agg["legislative_session__jurisdiction__name"]
            if agg["count"] > MAX_UPDATE:
                click.secho(
                    f"Too many bills to update for {state_name}: {agg['count']}, skipping",
                    fg="red",
                )
                missing_search = missing_search.exclude(
                    legislative_session__jurisdiction__name=state_name
                )
        print(f"{len(missing_search)} missing, updating")
    else:
        print(
            f"{state}: {len(all_bills)} bills, {len(missing_search)} without search results"
        )

    if n:
        missing_search = missing_search[: int(n)]
    else:
        n = len(missing_search)

    ids_to_update = []
    updated_count = 0

    # going to manage our own transactions here so we can save in chunks
    transaction.set_autocommit(False)

    for b in missing_search:
        ids_to_update.append(update_bill(b))
        updated_count += 1
        if updated_count % status_num == 0:
            print(f"{state}: updated {updated_count} out of {n}")
        if updated_count % checkpoint == 0:
            reindex(ids_to_update)
            transaction.commit()
            ids_to_update = []

    # be sure to reindex final set
    reindex(ids_to_update)
    transaction.commit()
    transaction.set_autocommit(True)
コード例 #7
0
    jurisdiction_id = get_jurisdiction_id(abbr)
    voters, sponsorships = get_unmatched(jurisdiction_id)
    update_objects(jurisdiction_id, voters, "vote", dry)
    update_objects(jurisdiction_id, sponsorships, "sponsorship", dry)


@click.command()
@click.argument("abbreviations", nargs=-1)
@click.option("--dump/--no-dump")
@click.option("--match/--no-match")
@click.option("--dry/--no-dry", default=True)
def process_unmatched(abbreviations, dump, match, dry):
    if not abbreviations:
        abbreviations = get_all_abbreviations()

    for abbr in abbreviations:
        if match:
            if dry:
                click.secho("dry run, nothing will be saved", fg="blue")
            try:
                check_historical_matches(abbr, dry=dry)
            except AbortTransaction as e:
                click.secho(f"{e}\ntransaction aborted!", fg="red")
        if dump:
            archive_leg_to_csv(abbr)


if __name__ == "__main__":
    init_django()
    process_unmatched()