def handle(self, *args, **options):
super(Command, self).handle(*args, **options)
self.missing_raw_files = []
self.empty_raw_files = []
self.unknown_raw_files = []
try:
last_complete_download = self.command_logs.filter(
command="downloadcalaccessrawdata", finish_datetime__isnull=False
).order_by("-finish_datetime")[0]
except IndexError:
self.failure("No complete downloads yet.")
self.log("Try: python manage.py updatecalaccessrawdata")
return
else:
self.raw_data_files = self.raw_data_files.filter(version=last_complete_download.version)
downloaded_release_datetime = last_complete_download.version.release_datetime
current_release_datetime = self.get_download_metadata()["last-modified"]
if current_release_datetime != downloaded_release_datetime:
since_new_release = naturaltime(current_release_datetime)
self.failure("A new version of CAL-ACCESS was " "released {}.".format(since_new_release))
self.log("Try: python manage.py updatecalaccessrawdata")
self.log("Analyzing loaded models")
if self.verbosity == 1:
model_list = progress.bar(get_model_list())
else:
model_list = get_model_list()
for model in model_list:
call_command("verifycalaccessrawfile", model.__name__, verbosity=self.verbosity)
self.num_download_files = self.raw_data_files.count()
self.num_clean_files = self.raw_data_files.filter(clean_records_count__gt=0).count()
self.num_loaded_files = self.raw_data_files.filter(load_records_count__gt=0).count()
total_raw_records = self.sum_count_column("download_records_count")
total_clean_records = self.sum_count_column("clean_records_count")
total_loaded_records = self.sum_count_column("load_records_count")
self.log("{0:.4%} of records cleaned".format(calc_percent(total_clean_records, total_raw_records)))
self.log("{0:.4%} of records loaded".format(calc_percent(total_loaded_records, total_raw_records)))
self.write_csv_results()
self.duration()
def test_model_methods(self):
"""
Test the methods that hook up with our models.
"""
version = models.RawDataVersion(release_datetime=datetime.now())
calaccess_raw.archive_directory_path(version, 'foobar.csv')
datafile = models.RawDataFile(version=version)
calaccess_raw.archive_directory_path(datafile, 'foobar.csv')
with self.assertRaises(TypeError):
calaccess_raw.archive_directory_path(models.RcptCd(), 'foobar.csv')
calaccess_raw.get_model_list()
def test_model_methods(self):
"""
Test the methods that hook up with our models.
"""
version = models.RawDataVersion(release_datetime=datetime.now())
calaccess_raw.archive_directory_path(version, 'foobar.csv')
datafile = models.RawDataFile(version=version)
calaccess_raw.archive_directory_path(datafile, 'foobar.csv')
with self.assertRaises(TypeError):
calaccess_raw.archive_directory_path(
models.RcptCd(),
'foobar.csv'
)
calaccess_raw.get_model_list()
def get_models(self):
models = []
for model in get_model_list():
if self in [x[0] for x in model().get_filing_forms_w_sections()]:
models.append(model)
return models
def handle(self, *args, **kwargs):
"""
Make it happen.
"""
self.set_options()
self.header(
"Creating GitHub issues for model models without a UNIQUE_KEY")
# Loop through all the models and find any fields without docs
missing_list = []
model_count = 0
for m in get_model_list():
model_count += 1
if not m.UNIQUE_KEY:
self.log("Missing: %s.%s" % (
m().klass_group,
m().klass_name,
))
missing_list.append(m)
# If everything is done, declare victory
missing_count = len(missing_list)
if not missing_count:
self.success("All %s models have a UNIQUE_KEY!" % missing_count)
return False
# If not, loop through the missing and create issues
self.log("- %s/%s (%d%%) of fields lack a UNIQUE_KEY" %
(intcomma(missing_count), intcomma(model_count),
calculate.percentage(missing_count, model_count)))
for model in missing_list[1:]:
self.create_issue(model)
def handle(self, *args, **kwargs):
"""
Make it happen.
"""
self.set_options()
self.header(
"Creating GitHub issues for model fields without documentation")
# Loop through all the models and find any fields without docs
field_count = 0
missing_list = []
for m in get_model_list():
field_list = m().get_field_list()
field_count += len(field_list)
for f in field_list:
if not self.has_docs(f):
missing_list.append((m, f))
# If everything is done, declare victory
if not missing_list:
self.success("All %s fields documented!" % field_count)
return False
# If not, loop through the missing and create issues
missing_count = len(missing_list)
self.log("- %s/%s (%d%%) of fields lack documentation" %
(intcomma(missing_count), intcomma(field_count),
calculate.percentage(missing_count, field_count)))
for model, field in missing_list[611:]:
# For now we are excluding the 'other' model module to
# avoid overkill
if model().klass_group != 'other':
self.create_issue(model, field)
def handle(self, *args, **kwargs):
"""
Make it happen.
"""
# Loop through all the models and find any fields without docs
missing_list = []
model_count = 0
for m in get_model_list():
model_count += 1
if m.UNIQUE_KEY is None:
self.log("Missing: %s.%s" % (
m().klass_group,
m().klass_name,
))
missing_list.append(m)
# If everything is done, declare victory
missing_count = len(missing_list)
if not missing_count:
self.success("All %s models have a UNIQUE_KEY!" % missing_count)
return False
# If not, loop through the missing and create issues
self.failure("%s/%s (%d%%) of models lack a UNIQUE_KEY" %
(intcomma(missing_count), model_count,
calculate.percentage(missing_count, model_count)))
def load(self):
"""
Loads the cleaned up csv files into the database.
"""
if self.verbosity:
self.header("Loading data files")
model_list = [
x for x in get_model_list()
if os.path.exists(x.objects.get_csv_path())
]
if self.resume_mode:
# get finished load command logs of last update
prev_loaded = [
x.file_name for x in self.log_record.called.filter(
command='loadcalaccessrawfile',
finish_datetime__isnull=False)
]
self.log("{} models already loaded.".format(len(prev_loaded)))
# remove these from model_list
model_list = [
x for x in model_list if x._meta.db_table not in prev_loaded
]
if self.verbosity:
model_list = progress.bar(model_list)
for model in model_list:
call_command(
"loadcalaccessrawfile",
model.__name__,
verbosity=self.verbosity,
keep_files=self.keep_files,
app_name=self.app_name,
)
def _test_choices(self, field_name):
"""
Verify that proper choices appear for the provided field.
"""
# Loop through the models
for m in get_model_list():
# And then through the fields
for f in m._meta.fields:
# Only test against the provided field name
if not f.name == field_name:
continue
# Pull out all the choices in that field
slug_list = []
if not f.choices:
warnings.warn("%s %s has no choices defined" %
(m.__name__, field_name))
for slug, name in f.choices:
# Make sure that each has a definition
# self.assertIsNot(name, '')
if not name:
warnings.warn("%s %s '%s' undefined" % (
m.__name__,
field_name,
slug,
))
slug_list.append(slug)
# The query the database and make sure everything in
# there has a matching definition in the choices
for v in m.objects.values(field_name).distinct():
self.assertIn(v, slug_list)
def _test_choices(self, field_name):
"""
Verify that proper choices appear for the provided field.
"""
# Loop through the models
for m in get_model_list():
# And then through the fields
for f in m._meta.fields:
# Only test against the provided field name
if not f.name == field_name:
continue
# Pull out all the choices in that field
slug_list = []
if not f.choices:
warnings.warn("%s %s has no choices defined" % (m.__name__, field_name))
for slug, name in f.choices:
# Make sure that each has a definition
# self.assertIsNot(name, '')
if not name:
warnings.warn("%s %s '%s' undefined" % (m.__name__, field_name, slug))
slug_list.append(slug)
# The query the database and make sure everything in
# there has a matching definition in the choices
for value, count in m.objects.values_list(field_name).annotate(Count(field_name)):
warnings.warn("'%s' %s code undefined on %s model" % (value, field_name, m.__name__))
class CalAccessFileDownloadsSitemap(AbstractSitemapView):
"""
A machine-readable list of all CalAccess file archive download pages.
"""
build_path = 'raw-file-downloads-sitemap.xml'
template_name = 'calaccess_website/sitemaps/calaccess-file-downloads-sitemap.xml'
queryset = get_model_list()
def test_model_str(self):
"""
Verify that __str__ methods exist and work for all models.
"""
for m in get_model_list():
m().__str__()
self.assertNotEqual(m().__str__(), '%s object' % m.__name__)
def test_model_doc(self):
"""
Verify that __doc__ methods exist and work for all models.
"""
for m in get_model_list():
if m().__doc__.startswith(m.__name__):
warnings.warn("%s __doc__ undefined" % (m.__name__))
def test_model_doc(self):
"""
Verify that __doc__ methods exist and work for all models.
"""
for m in get_model_list():
if m().__doc__.startswith(m.__name__):
warnings.warn("%s __doc__ undefined" % (m.__name__))
def test_model_str(self):
"""
Verify that __str__ methods exist and work for all models.
"""
for m in get_model_list():
m().__str__()
self.assertNotEqual(m().__str__(), '%s object' % m.__name__)
def handle(self, *args, **kwargs):
"""
Make it happen.
"""
# Loop through all the models and find any fields without docs
missing_list = []
model_count = 0
for m in get_model_list():
model_count += 1
if not m.UNIQUE_KEY:
self.log("Missing: %s.%s" % (
m().klass_group,
m().klass_name,
)
)
missing_list.append(m)
# If everything is done, declare victory
missing_count = len(missing_list)
if not missing_count:
self.success("All %s models have a UNIQUE_KEY!" % missing_count)
return False
# If not, loop through the missing and create issues
self.failure(
"%s/%s (%d%%) of models lack a UNIQUE_KEY" % (
intcomma(missing_count),
model_count,
calculate.percentage(missing_count, model_count)
)
)
def test_model_unique_key(self):
"""
Verify that each model has a UNIQUE_KEY attribute set.
"""
results = []
# Loop through the models
for m in get_model_list():
# Verify the model has a UNIQUE_KEY attribute
exists = m().UNIQUE_KEY is not None
# Verify that the field names are not lowercase
lowercases = sum([
1 for k in m().get_unique_key_list()
if re.search("[a-z]{1,}", k)
]) or 0
if lowercases:
exists = False
# Verify that the fields actually exist
missing = [
f for f in m().get_unique_key_list()
if not hasattr(m(), f.lower())
and not hasattr(m(),
f.lower() + "_id")
]
if missing:
exists = False
# Pass out results
results.append([m().klass_group, m.__name__, exists])
# Print results
self.attr_test_output("model", "UNIQUE_KEY", results)
def test_csv_gettrs(self):
"""
Verify that get_csv_name methods work for all models.
"""
for m in get_model_list():
self.assertEqual(m.objects.get_csv_name(), m().get_csv_name())
self.assertEqual(m.objects.get_csv_path(), m().get_csv_path())
def handle(self, *args, **kwargs):
"""
Make it happen.
"""
# Loop through all the models and find any fields without docs
field_count = 0
missing_list = []
for m in get_model_list():
field_list = m().get_field_list()
field_count += len(field_list)
for f in field_list:
if not self.has_docs(f):
self.log("Missing: %s.%s.%s" %
(m().klass_group, m().klass_name, f))
missing_list.append((m, f))
# If everything is done, declare victory
if not missing_list:
self.success("All %s fields documented!" % field_count)
return False
# If not, loop through the missing and create issues
missing_count = len(missing_list)
self.failure("%s/%s (%d%%) of fields lack documentation" %
(intcomma(missing_count), intcomma(field_count),
calculate.percentage(missing_count, field_count)))
def test_csv_gettrs(self):
"""
Verify that get_csv_name methods work for all models.
"""
for m in get_model_list():
self.assertEqual(m.objects.get_csv_name(), m().get_csv_name())
self.assertEqual(m.objects.get_csv_path(), m().get_csv_path())
def load(self):
"""
Loads the cleaned up csv files into the database
"""
if self.verbosity:
self.header("Loading data files")
model_list = [
x for x in get_model_list() if os.path.exists(x.objects.get_csv_path())
]
if self.resume_mode:
# get finished load command logs of last update
prev_loaded = [
x.file_name
for x in self.log_record.called.filter(
command='loadcalaccessrawfile',
finish_datetime__isnull=False
)
]
self.log("{} models already loaded.".format(len(prev_loaded)))
# remove these from model_list
model_list = [x for x in model_list if x._meta.db_table not in prev_loaded]
if self.verbosity:
model_list = progress.bar(model_list)
for model in model_list:
call_command(
"loadcalaccessrawfile",
model.__name__,
verbosity=self.verbosity,
keep_files=self.keep_files,
app_name=self.app_name,
)
def handle(self, *args, **kwargs):
"""
Make it happen.
"""
# Loop through all the models and find any fields without docs
field_count = 0
missing_list = []
for m in get_model_list():
field_list = m().get_field_list()
field_count += len(field_list)
for f in field_list:
if not self.has_docs(f):
self.log("Missing: %s.%s.%s" % (
m().klass_group,
m().klass_name,
f
)
)
missing_list.append((m, f))
# If everything is done, declare victory
if not missing_list:
self.success("All %s fields documented!" % field_count)
return False
# If not, loop through the missing and create issues
missing_count = len(missing_list)
self.failure(
"%s/%s (%d%%) of fields lack documentation" % (
intcomma(missing_count),
intcomma(field_count),
calculate.percentage(missing_count, field_count)
)
)
def handle(self, *args, **kwargs):
self.header("Totaling CAL-ACCESS tables and rows")
model_list = get_model_list()
self.log(" %s total tables" % len(model_list))
record_count = 0
for m in model_list:
record_count += m.objects.count()
self.log(" %s total records" % intcomma(record_count))
def test_models(self):
"""
Make sure all the models have admins.
"""
model_list = [m.__name__ for m in get_model_list()]
admin_list = [a.replace("Admin", "") for a in admin.__all__]
missing = set(model_list).difference(admin_list)
self.assertEqual(missing, set([]))
def handle(self, *args, **kwargs):
self.header("Totaling CAL-ACCESS tables and rows")
model_list = get_model_list()
self.log(" %s total tables" % len(model_list))
record_count = 0
for m in model_list:
record_count += m.objects.count()
self.log(" %s total records" % intcomma(record_count))
def _test_choices(self, field_name):
"""
Verify that proper choices appear for the provided field.
"""
message_list = []
# Loop through the models
for m in get_model_list():
# And then through the fields
for f in m._meta.fields:
# Only test against the provided field name
if not f.name == field_name:
continue
if not f.choices:
message_list.append((
m.__name__,
field_name,
"Has no CHOICES defined"
))
if not f.documentcloud_pages:
message_list.append((
m.__name__,
field_name,
"Has no `documentcloud_pages_urls` defined"
))
# Pull out all the choices in that field
slug_list = []
for slug, name in f.choices:
# Make sure that each has a definition
if not name:
message_list.append((
m.__name__,
field_name,
"Value '%s' undefined in CHOICES" % slug
))
if name.lower() == 'unknown':
message_list.append((
m.__name__,
field_name,
"Value '%s' defined as 'Unknown'" % slug
))
slug_list.append(slug)
# The query the database and make sure everything in
# there has a matching definition in the choices
for value, count in m.objects.values_list(
field_name,
).annotate(Count(field_name)):
message_list.append((
m.__name__,
field_name,
"Value '%s' in database but not in CHOICES" % value
))
return message_list
def handle(self, *args, **kwargs):
self.docs_dir = os.path.join(
settings.REPO_DIR,
'docs'
)
self.target_path = os.path.join(self.docs_dir, 'models.rst')
model_list = sorted(get_model_list(), key=lambda x:x().klass_name)
group_list = {}
empty_files = []
for m in model_list:
# add doc_id, page_url list to each model
m.docs = {}
for doc in m.DOCUMENTCLOUD_PAGES:
doc_title = doc.get_doc_data()['title']
try:
m.docs[doc_title].extend(
zip(
doc.get_page_urls(), doc.get_thumbnail_urls()
)
)
except KeyError:
m.docs[doc_title] = list(zip(
doc.get_page_urls(), doc.get_thumbnail_urls()
))
# add choice field list to model
m.choice_fields = []
for field in m._meta.fields:
if len(field.choices) > 0:
# add doc_id, page_url list to each choice field
field.docs = {}
for doc in field.documentcloud_pages:
doc_title = doc.get_doc_data()['title']
try:
field.docs[doc_title].extend(
zip(
doc.get_page_urls(), doc.get_thumbnail_urls()
)
)
except KeyError:
field.docs[doc_title] = list(zip(
doc.get_page_urls(), doc.get_thumbnail_urls()
))
m.choice_fields.append(field)
try:
group_list[m().klass_group].append(m)
except KeyError:
group_list[m().klass_group] = [m]
if m.objects.count() == 0:
empty_files.append(m)
group_list = sorted(group_list.items(), key=lambda x:x[0])
context = {
'group_list': group_list,
'model_count': len(model_list),
'empty_files': empty_files,
}
rendered = render_to_string('toolbox/models.rst', context)
with open(self.target_path, 'w') as target_file:
target_file.write(rendered)
def test_model_unique_key(self):
"""
Verify that each model has a UNIQUE_KEY attribute set.
"""
results = []
for m in get_model_list():
exists = m().UNIQUE_KEY is not None
results.append([m.__name__, exists])
self.attr_test_output("model", "UNIQUE_KEY", results)
def load(self):
"""
Loads the cleaned up csv files into the database
"""
if self.verbosity:
self.header("Loading data files")
for model in get_model_list():
call_command("loadcalaccessrawfile", model.__name__)
def test_model_unique_key(self):
"""
Verify that each model has a UNIQUE_KEY attribute set.
"""
results = []
for m in get_model_list():
exists = m().UNIQUE_KEY is not None
results.append([m.__name__, exists])
self.attr_test_output("model", "UNIQUE_KEY", results)
def get_context_data(self, **kwargs):
context = super(CalAccessFileList, self).get_context_data(**kwargs)
model_list = get_model_list()
context['model_list'] = model_list
context['title'] = 'Raw files'
context[
'description'] = "Definitions, record layouts and data dictionaries for the {} raw \
files released from the California Secretary of State's CAL-ACCESS database. For experts only.".format(
len(model_list))
return context
def get_models(self):
"""
Returns all the CAL-ACCESS models connected with this form.
"""
models = []
for model in get_model_list():
if self in [x[0] for x in model().get_filing_forms_w_sections()]:
models.append(model)
return models
def model(self):
"""
Returns the RawDataFile's corresponding CalAccess database model object.
"""
try:
return [
m for m in get_model_list() if m().db_table == self.file_name
][0]
except IndexError:
return None
def model(self):
"""
Returns the RawDataFile's corresponding CalAccess database model object.
"""
try:
return [
m for m in get_model_list() if m().db_table == self.file_name
][0]
except IndexError:
return None
def get_models(self):
"""
Returns all the CAL-ACCESS models connected with this form.
"""
models = []
for model in get_model_list():
if self in [x[0] for x in model().get_filing_forms_w_sections()]:
models.append(model)
return models
def handle(self, *args, **options):
super(Command, self).handle(*args, **options)
model_list = sorted(
get_model_list(),
key=lambda x: (x().klass_group, x().klass_name)
)
results = []
for m in model_list:
if self.verbosity > 1:
self.log(
" Verifying {0}.{1} fields".format(
m().klass_group,
m.__name__,
)
)
for f in m._meta.fields:
if f.choices:
for value, count in m.objects.order_by().values_list(
f.name,
).annotate(Count(f.name)):
if (
value not in [x[0] for x in f.choices] and
value != '' and
value is not None
):
if self.verbosity > 2:
self.failure(
" Undefined value for {0}: {1} ({2} occurrences)".format(
f.name,
value,
count
)
)
results.append((
m().klass_group,
m.__name__,
f.name,
value,
count,
))
if len(results) > 0:
self.failure("{} undefined choice field values".format(len(results)))
table = agate.Table(
results,
['group', 'model', 'field', 'undefined_value', 'occurrences']
)
table.print_table(max_rows=None, max_column_width=50)
else:
self.success("No undefined choice field values")
def test_model_str(self):
"""
Verify that __str__ methods exist and work for all models.
"""
results = []
for m in get_model_list():
# Test that works
m().__str__()
# Make sure that it is customized and not the Django defalult
is_custom = m().__str__() != '%s object' % m.__name__
results.append([m.__name__, is_custom])
self.attr_test_output("model", "__str__", results)
def test_model_doc(self):
"""
Verify that __doc__ methods exist and work for all models.
"""
results = []
for m in get_model_list():
if m().__doc__.startswith(m.__name__):
exists = False
else:
exists = True
results.append([m.__name__, exists])
self.attr_test_output("model", "__doc__", results)
def test_model_documentcloud_pages(self):
"""
Verify that each model has DOCUMENTCLOUD_PAGES defined
"""
results = []
for m in get_model_list():
if m().DOCUMENTCLOUD_PAGES:
exists = True
else:
exists = False
results.append([m().klass_group, m.__name__, exists])
self.attr_test_output("model", "DOCUMENTCLOUD_PAGES", results)
def test_model_str(self):
"""
Verify that __str__ methods exist and work for all models.
"""
results = []
for m in get_model_list():
# Test that works
m().__str__()
# Make sure that it is customized and not the Django defalult
is_custom = m().__str__() != '%s object' % m.__name__
results.append([m.__name__, is_custom])
self.attr_test_output("model", "__str__", results)
def test_model_documentcloud_pages(self):
"""
Verify that each model has DOCUMENTCLOUD_PAGES defined
"""
results = []
for m in get_model_list():
if m().DOCUMENTCLOUD_PAGES:
exists = True
else:
exists = False
results.append([m().klass_group, m.__name__, exists])
self.attr_test_output("model", "DOCUMENTCLOUD_PAGES", results)
def test_model_doc(self):
"""
Verify that __doc__ methods exist and work for all models.
"""
results = []
for m in get_model_list():
if m().__doc__.startswith(m.__name__):
exists = False
else:
exists = True
results.append([m.__name__, exists])
self.attr_test_output("model", "__doc__", results)
def test_choices(self):
"""
Verify that valid choices are available for all expected fields on all models.
"""
# substrings that appear in choice fields
choice_field_strs = [
'_cd',
'_code',
'_type',
'status',
'_lvl',
'reportname',
'form_id',
]
exceptions = [
'LookupCodesCd.code_type',
'S497Cd.sup_off_cd',
'FilerStatusTypesCd.status_type',
'FilerStatusTypesCd.status_desc',
'FilerTypesCd.filer_type',
]
results = []
model_list = sorted(get_model_list(),
key=lambda x: (x().klass_group, x().klass_name))
for m in model_list:
for f in m._meta.fields:
if (any(x in f.name
for x in choice_field_strs) and f.name != 'memo_code'
and f.__class__ is not ForeignKeyField
and '{}.{}'.format(m().klass_name,
f.name) not in exceptions):
if not f.choices:
results.append((m().klass_group, m.__name__, f.name,
"Has no CHOICES defined"))
if not f.documentcloud_pages:
results.append(
(m().klass_group, m.__name__, f.name,
"Has no `documentcloud_pages` defined"))
# Pull out all the choices in that field
for slug, name in f.choices:
# Make sure that each has a definition
if not name or name == '':
results.append(
(m().klass_group, m.__name__, f.name,
"Value '%s' undefined in CHOICES" % slug))
table = agate.Table(results, ['group', 'model', 'field', 'message'])
table.print_table(max_rows=None, max_column_width=50)
def test_field_docs(self):
"""
Verify that all fields have verbose_name or help_text documentation.
"""
for m in get_model_list():
for f in m().get_field_list():
if f.name == "id":
continue
if f.help_text:
continue
if f.__dict__["_verbose_name"]:
continue
warnings.warn("%s.%s field undocumented" % (m().klass_name, f.name))
def handle(self, *args, **options):
super(Command, self).handle(*args, **options)
"""
Connect to Github using token stored in environment, loop over model fields, and \
create an issue for any choice field missing
"""
self.dry_run = options["dry_run"]
# set up connect to Github account
self.gh = Github(os.getenv('GITHUB_TOKEN'))
self.org = self.gh.get_organization("california-civic-data-coalition")
self.repo = self.org.get_repo("django-calaccess-raw-data")
self.labels = [
self.repo.get_label("small"),
self.repo.get_label("documentation"),
self.repo.get_label("enhancement"),
]
self.header(
"Creating GitHub issues for model choice fields"
)
choice_field_strs = [
'_cd',
'_code',
'type',
'status',
'_lvl',
'reportname',
'form_id',
]
excluded_fields = [
'LookupCodesCd.code_type',
'S497Cd.sup_off_cd',
]
model_list = sorted(
get_model_list(),
key=lambda x: (x().klass_group, x().klass_name)
)
for m in model_list:
for f in m._meta.fields:
if (
any(x in f.name for x in choice_field_strs) and
f.name != 'memo_code' and
f.__class__ is not models.ForeignKey and
'{}.{}'.format(m().klass_name, f.name) not in excluded_fields
):
# make an issue for every choice field missing docs
# includes those that are also missing choices
if not f.documentcloud_pages:
self.create_issue(f)
def load(self):
"""
Loads the cleaned up csv files into the database
"""
if self.verbosity:
self.header("Loading data files")
model_list = get_model_list()
for model in progress.bar(model_list):
call_command(
"loadcalaccessrawfile",
model.__name__,
verbosity=self.verbosity,
)
def test_filing_forms(self):
"""
Verify that each model with a form_type or form_id field has FILING_FORMS defined
"""
results = []
for m in get_model_list():
field_names = m._meta.get_all_field_names()
if 'form_type' in field_names or 'form_id' in field_names:
if m().FILING_FORMS:
exists = True
else:
exists = False
results.append([m().klass_group, m.__name__, exists])
self.attr_test_output("model", "FILING_FORMS", results)
def test_field_docs(self):
"""
Verify that all fields have verbose_name or help_text documentation.
"""
for m in get_model_list():
for f in m().get_field_list():
if f.name == 'id':
continue
if f.help_text:
continue
if f.__dict__['_verbose_name']:
continue
warnings.warn("%s.%s field undocumented" %
(m().klass_name, f.name))
def handle(self, *args, **kwargs):
self.docs_dir = os.path.join(settings.REPO_DIR, 'docs')
self.target_path = os.path.join(self.docs_dir, 'models.rst')
model_list = sorted(get_model_list(), key=lambda x: x().klass_name)
group_list = {}
empty_files = []
for m in model_list:
# add doc_id, page_url list to each model
m.docs = {}
for doc in m.DOCUMENTCLOUD_PAGES:
doc_title = doc.get_doc_data()['title']
try:
m.docs[doc_title].extend(
zip(doc.get_page_urls(), doc.get_thumbnail_urls()))
except KeyError:
m.docs[doc_title] = list(
zip(doc.get_page_urls(), doc.get_thumbnail_urls()))
# add choice field list to model
m.choice_fields = []
for field in m._meta.fields:
if len(field.choices) > 0:
# add doc_id, page_url list to each choice field
field.docs = {}
for doc in field.documentcloud_pages:
doc_title = doc.get_doc_data()['title']
try:
field.docs[doc_title].extend(
zip(doc.get_page_urls(),
doc.get_thumbnail_urls()))
except KeyError:
field.docs[doc_title] = list(
zip(doc.get_page_urls(),
doc.get_thumbnail_urls()))
m.choice_fields.append(field)
try:
group_list[m().klass_group].append(m)
except KeyError:
group_list[m().klass_group] = [m]
if m.objects.count() == 0:
empty_files.append(m)
group_list = sorted(group_list.items(), key=lambda x: x[0])
context = {
'group_list': group_list,
'model_count': len(model_list),
'empty_files': empty_files,
}
rendered = render_to_string('toolbox/models.rst', context)
with open(self.target_path, 'w') as target_file:
target_file.write(rendered)
def test_field_help_text(self):
"""
Verify that all fields have help_text documentation.
"""
results = []
for m in get_model_list():
for f in m().get_field_list():
if f.name == 'id':
continue
if f.help_text:
exists = True
else:
exists = False
results.append(("%s.%s" % (m.__name__, f.name), exists))
self.attr_test_output("field", "help_text", results)
def handle(self, *args, **options):
docs = set()
cached = set()
for m in get_model_list():
docs.update(set(m().DOCUMENTCLOUD_PAGES))
for form in all_filing_forms:
docs.add(form.documentcloud)
for doc in docs:
if doc and doc.metadata_filename not in cached:
logger.debug('Caching {}'.format(doc.metadata_filename))
doc._cache_metadata()
cached.add(doc.metadata_filename)
def test_field_verbose_name(self):
"""
Verify that all fields have verbose_name documentation.
"""
results = []
for m in get_model_list():
for f in m().get_field_list():
if f.name == 'id':
continue
if f.__dict__['_verbose_name']:
exists = True
else:
exists = False
results.append(("%s.%s" % (m.__name__, f.name), exists))
self.attr_test_output("field", "verbose_name", results)
def test_field_help_text(self):
"""
Verify that all fields have help_text documentation.
"""
results = []
for m in get_model_list():
for f in m().get_field_list():
if f.name == 'id':
continue
if f.help_text:
exists = True
else:
exists = False
results.append(("%s.%s" % (m.__name__, f.name), exists))
self.attr_test_output("field", "help_text", results)
def test_field_verbose_name(self):
"""
Verify that all fields have verbose_name documentation.
"""
results = []
for m in get_model_list():
for f in m().get_field_list():
if f.name == 'id':
continue
if f.__dict__['_verbose_name']:
exists = True
else:
exists = False
results.append(("%s.%s" % (m.__name__, f.name), exists))
self.attr_test_output("field", "verbose_name", results)
def get_context_data(self):
model_list = calaccess_raw.get_model_list()
form_list = FORMS
object_list = [
dict(
name='Processed files from the California Civic Data Coalition',
description="Definitions, record layouts and data dictionaries for the \
processed data files released by the California Civic Data Coalition. Recommended for beginners and regular use.",
url=reverse("ccdc_file_list"),
),
dict(
name='Raw files from CAL-ACCESS',
description="Definitions, record layouts and data dictionaries for the {} raw \
files released from the California Secretary of State's CAL-ACCESS database. For experts only.".format(len(model_list)),
url=reverse("calaccess_file_list"),
),
dict(
name='Official CAL-ACCESS forms',
description="Descriptions, samples and other documentation for \
the {} forms that campaigns and lobbyists use to disclose activity to California's Secretary of \
State.".format(len(form_list)),
url=reverse("form_list"),
),
dict(
name='Official CAL-ACCESS documentation',
description="The jumbled, fragmentary and unreliable documentation \
for the CAL-ACCESS database provided by California's Secretary of State. For more authoritative \
information refer to the materials above.",
url=reverse("official_documentation"),
),
dict(
name='Technical documentation',
description="A user manual for the collection of \
California Civic Data Coalition applications that power this site. For developers only.",
url="//django-calaccess.californiacivicdata.org"
),
dict(
name="Frequently asked questions",
description="Answers to common questions about this project and the underlying CAL-ACCESS database.",
url=reverse("faq"),
),
]
return {
'object_list': object_list,
'title': "Documentation",
'description': "How to work with our data tracking campaign finance and lobbying activity in \
California politics."
}
def load(self):
"""
Loads the cleaned up csv files into the database
"""
if self.verbosity:
self.header("Loading data files")
model_list = get_model_list()
if self.verbosity:
model_list = progress.bar(model_list)
for model in model_list:
call_command(
"loadcalaccessrawfile",
model.__name__,
verbosity=self.verbosity,
)
def _test_choices(self, field_name):
"""
Verify that proper choices appear for the provided field.
"""
message_list = []
# Loop through the models
for m in get_model_list():
# And then through the fields
for f in m._meta.fields:
# Only test against the provided field name
if not f.name == field_name:
continue
if not f.choices:
message_list.append((m().klass_group, m.__name__,
field_name, "Has no CHOICES defined"))
if not f.documentcloud_pages:
message_list.append(
(m().klass_group, m.__name__, field_name,
"Has no `documentcloud_pages` defined"))
# Pull out all the choices in that field
slug_list = []
for slug, name in f.choices:
# Make sure that each has a definition
if not name:
message_list.append(
(m().klass_group, m.__name__, field_name,
"Value '%s' undefined in CHOICES" % slug))
if name.lower() == 'unknown':
message_list.append(
(m().klass_group, m.__name__, field_name,
"Value '%s' defined as 'Unknown'" % slug))
slug_list.append(slug)
# The query the database and make sure everything in
# there has a matching definition in the choices
for value, count in m.objects.values_list(
field_name, ).annotate(Count(field_name)):
message_list.append(
(m().klass_group, m.__name__, field_name,
"Value '%s' in database but not in CHOICES" % value))
return message_list
def handle(self, *args, **kwargs):
self.docs_dir = os.path.join(settings.REPO_DIR, "docs")
self.target_path = os.path.join(self.docs_dir, "models.rst")
model_list = sorted(get_model_list(), key=lambda x: x().klass_name)
group_list = {}
empty_files = []
for m in model_list:
try:
group_list[m().klass_group].append(m)
except KeyError:
group_list[m().klass_group] = [m]
if m.objects.count() == 0:
empty_files.append(m)
group_list = sorted(group_list.items(), key=lambda x: x[0])
context = {"group_list": group_list, "model_count": len(model_list), "empty_files": empty_files}
rendered = render_to_string("toolbox/models.rst", context)
with open(self.target_path, "w") as target_file:
target_file.write(rendered)
