def __init__(self, abbr, settings):
self.http_whitelist = tuple(settings.get("http_whitelist", []))
self.expected = get_expected_districts(settings, abbr)
self.valid_parties = set(settings["parties"])
self.errors = defaultdict(list)
self.warnings = defaultdict(list)
self.person_count = 0
self.retired_count = 0
self.org_count = 0
self.missing_person_id = 0
self.missing_person_id_percent = 0
self.role_types = defaultdict(int)
self.parent_types = defaultdict(int)
self.person_mapping = {}
self.parties = Counter()
self.contact_counts = Counter()
self.id_counts = Counter()
self.optional_fields = Counter()
self.extra_counts = Counter()
# role type -> district -> person
self.active_legislators = defaultdict(lambda: defaultdict(list))
# field name -> value -> person
self.duplicate_values = defaultdict(lambda: defaultdict(list))
self.legacy_districts = legacy_districts(abbr=abbr)
self.municipalities = [m["id"] for m in load_municipalities(abbr=abbr)]
for m in self.municipalities:
if not JURISDICTION_RE.match(m):
raise ValueError(f"invalid municipality id {m}")
def __init__(self, abbr, settings):
self.http_whitelist = tuple(settings.get("http_whitelist", []))
self.expected = get_expected_districts(settings, abbr)
self.valid_parties = set(settings["parties"])
self.errors = defaultdict(list)
self.warnings = defaultdict(list)
# role type -> district -> filename
self.active_legislators = defaultdict(lambda: defaultdict(list))
# field name -> value -> filename
self.duplicate_values = defaultdict(lambda: defaultdict(list))
self.legacy_districts = legacy_districts(abbr=abbr)
self.municipalities = [m["id"] for m in load_municipalities(abbr=abbr)]
for m in self.municipalities:
if not JURISDICTION_RE.match(m):
raise ValueError(f"invalid municipality id {m}")
def to_database(abbreviations, purge, safe):
"""
Sync YAML files to DB.
"""
init_django()
create_parties()
if not abbreviations:
abbreviations = get_all_abbreviations()
for abbr in abbreviations:
click.secho("==== {} ====".format(abbr), bold=True)
directory = get_data_dir(abbr)
jurisdiction_id = get_jurisdiction_id(abbr)
municipalities = load_municipalities(abbr)
with transaction.atomic():
create_municipalities(municipalities)
person_files = (
glob.glob(os.path.join(directory, "legislature/*.yml")) +
glob.glob(os.path.join(directory, "executive/*.yml")) +
glob.glob(os.path.join(directory, "municipalities/*.yml")) +
glob.glob(os.path.join(directory, "retired/*.yml")))
committee_files = glob.glob(
os.path.join(directory, "organizations/*.yml"))
if safe:
click.secho("running in safe mode, no changes will be made",
fg="magenta")
try:
with transaction.atomic():
load_directory(person_files,
"person",
jurisdiction_id,
purge=purge)
load_directory(committee_files,
"organization",
jurisdiction_id,
purge=purge)
if safe:
click.secho("ran in safe mode, no changes were made",
fg="magenta")
raise CancelTransaction()
except CancelTransaction:
sys.exit(1)