def handle(self, *args, **options): super(Command, self).handle(*args, **options) self.missing_raw_files = [] self.empty_raw_files = [] self.unknown_raw_files = [] try: last_complete_download = self.command_logs.filter( command="downloadcalaccessrawdata", finish_datetime__isnull=False ).order_by("-finish_datetime")[0] except IndexError: self.failure("No complete downloads yet.") self.log("Try: python manage.py updatecalaccessrawdata") return else: self.raw_data_files = self.raw_data_files.filter(version=last_complete_download.version) downloaded_release_datetime = last_complete_download.version.release_datetime current_release_datetime = self.get_download_metadata()["last-modified"] if current_release_datetime != downloaded_release_datetime: since_new_release = naturaltime(current_release_datetime) self.failure("A new version of CAL-ACCESS was " "released {}.".format(since_new_release)) self.log("Try: python manage.py updatecalaccessrawdata") self.log("Analyzing loaded models") if self.verbosity == 1: model_list = progress.bar(get_model_list()) else: model_list = get_model_list() for model in model_list: call_command("verifycalaccessrawfile", model.__name__, verbosity=self.verbosity) self.num_download_files = self.raw_data_files.count() self.num_clean_files = self.raw_data_files.filter(clean_records_count__gt=0).count() self.num_loaded_files = self.raw_data_files.filter(load_records_count__gt=0).count() total_raw_records = self.sum_count_column("download_records_count") total_clean_records = self.sum_count_column("clean_records_count") total_loaded_records = self.sum_count_column("load_records_count") self.log("{0:.4%} of records cleaned".format(calc_percent(total_clean_records, total_raw_records))) self.log("{0:.4%} of records loaded".format(calc_percent(total_loaded_records, total_raw_records))) self.write_csv_results() self.duration()
def test_model_methods(self): """ Test the methods that hook up with our models. """ version = models.RawDataVersion(release_datetime=datetime.now()) calaccess_raw.archive_directory_path(version, 'foobar.csv') datafile = models.RawDataFile(version=version) calaccess_raw.archive_directory_path(datafile, 'foobar.csv') with self.assertRaises(TypeError): calaccess_raw.archive_directory_path(models.RcptCd(), 'foobar.csv') calaccess_raw.get_model_list()
def test_model_methods(self): """ Test the methods that hook up with our models. """ version = models.RawDataVersion(release_datetime=datetime.now()) calaccess_raw.archive_directory_path(version, 'foobar.csv') datafile = models.RawDataFile(version=version) calaccess_raw.archive_directory_path(datafile, 'foobar.csv') with self.assertRaises(TypeError): calaccess_raw.archive_directory_path( models.RcptCd(), 'foobar.csv' ) calaccess_raw.get_model_list()
def get_models(self): models = [] for model in get_model_list(): if self in [x[0] for x in model().get_filing_forms_w_sections()]: models.append(model) return models
def handle(self, *args, **kwargs): """ Make it happen. """ self.set_options() self.header( "Creating GitHub issues for model models without a UNIQUE_KEY") # Loop through all the models and find any fields without docs missing_list = [] model_count = 0 for m in get_model_list(): model_count += 1 if not m.UNIQUE_KEY: self.log("Missing: %s.%s" % ( m().klass_group, m().klass_name, )) missing_list.append(m) # If everything is done, declare victory missing_count = len(missing_list) if not missing_count: self.success("All %s models have a UNIQUE_KEY!" % missing_count) return False # If not, loop through the missing and create issues self.log("- %s/%s (%d%%) of fields lack a UNIQUE_KEY" % (intcomma(missing_count), intcomma(model_count), calculate.percentage(missing_count, model_count))) for model in missing_list[1:]: self.create_issue(model)
def handle(self, *args, **kwargs): """ Make it happen. """ self.set_options() self.header( "Creating GitHub issues for model fields without documentation") # Loop through all the models and find any fields without docs field_count = 0 missing_list = [] for m in get_model_list(): field_list = m().get_field_list() field_count += len(field_list) for f in field_list: if not self.has_docs(f): missing_list.append((m, f)) # If everything is done, declare victory if not missing_list: self.success("All %s fields documented!" % field_count) return False # If not, loop through the missing and create issues missing_count = len(missing_list) self.log("- %s/%s (%d%%) of fields lack documentation" % (intcomma(missing_count), intcomma(field_count), calculate.percentage(missing_count, field_count))) for model, field in missing_list[611:]: # For now we are excluding the 'other' model module to # avoid overkill if model().klass_group != 'other': self.create_issue(model, field)
def handle(self, *args, **kwargs): """ Make it happen. """ # Loop through all the models and find any fields without docs missing_list = [] model_count = 0 for m in get_model_list(): model_count += 1 if m.UNIQUE_KEY is None: self.log("Missing: %s.%s" % ( m().klass_group, m().klass_name, )) missing_list.append(m) # If everything is done, declare victory missing_count = len(missing_list) if not missing_count: self.success("All %s models have a UNIQUE_KEY!" % missing_count) return False # If not, loop through the missing and create issues self.failure("%s/%s (%d%%) of models lack a UNIQUE_KEY" % (intcomma(missing_count), model_count, calculate.percentage(missing_count, model_count)))
def load(self): """ Loads the cleaned up csv files into the database. """ if self.verbosity: self.header("Loading data files") model_list = [ x for x in get_model_list() if os.path.exists(x.objects.get_csv_path()) ] if self.resume_mode: # get finished load command logs of last update prev_loaded = [ x.file_name for x in self.log_record.called.filter( command='loadcalaccessrawfile', finish_datetime__isnull=False) ] self.log("{} models already loaded.".format(len(prev_loaded))) # remove these from model_list model_list = [ x for x in model_list if x._meta.db_table not in prev_loaded ] if self.verbosity: model_list = progress.bar(model_list) for model in model_list: call_command( "loadcalaccessrawfile", model.__name__, verbosity=self.verbosity, keep_files=self.keep_files, app_name=self.app_name, )
def _test_choices(self, field_name): """ Verify that proper choices appear for the provided field. """ # Loop through the models for m in get_model_list(): # And then through the fields for f in m._meta.fields: # Only test against the provided field name if not f.name == field_name: continue # Pull out all the choices in that field slug_list = [] if not f.choices: warnings.warn("%s %s has no choices defined" % (m.__name__, field_name)) for slug, name in f.choices: # Make sure that each has a definition # self.assertIsNot(name, '') if not name: warnings.warn("%s %s '%s' undefined" % ( m.__name__, field_name, slug, )) slug_list.append(slug) # The query the database and make sure everything in # there has a matching definition in the choices for v in m.objects.values(field_name).distinct(): self.assertIn(v, slug_list)
def _test_choices(self, field_name): """ Verify that proper choices appear for the provided field. """ # Loop through the models for m in get_model_list(): # And then through the fields for f in m._meta.fields: # Only test against the provided field name if not f.name == field_name: continue # Pull out all the choices in that field slug_list = [] if not f.choices: warnings.warn("%s %s has no choices defined" % (m.__name__, field_name)) for slug, name in f.choices: # Make sure that each has a definition # self.assertIsNot(name, '') if not name: warnings.warn("%s %s '%s' undefined" % (m.__name__, field_name, slug)) slug_list.append(slug) # The query the database and make sure everything in # there has a matching definition in the choices for value, count in m.objects.values_list(field_name).annotate(Count(field_name)): warnings.warn("'%s' %s code undefined on %s model" % (value, field_name, m.__name__))
class CalAccessFileDownloadsSitemap(AbstractSitemapView): """ A machine-readable list of all CalAccess file archive download pages. """ build_path = 'raw-file-downloads-sitemap.xml' template_name = 'calaccess_website/sitemaps/calaccess-file-downloads-sitemap.xml' queryset = get_model_list()
def test_model_str(self): """ Verify that __str__ methods exist and work for all models. """ for m in get_model_list(): m().__str__() self.assertNotEqual(m().__str__(), '%s object' % m.__name__)
def test_model_doc(self): """ Verify that __doc__ methods exist and work for all models. """ for m in get_model_list(): if m().__doc__.startswith(m.__name__): warnings.warn("%s __doc__ undefined" % (m.__name__))
def handle(self, *args, **kwargs): """ Make it happen. """ # Loop through all the models and find any fields without docs missing_list = [] model_count = 0 for m in get_model_list(): model_count += 1 if not m.UNIQUE_KEY: self.log("Missing: %s.%s" % ( m().klass_group, m().klass_name, ) ) missing_list.append(m) # If everything is done, declare victory missing_count = len(missing_list) if not missing_count: self.success("All %s models have a UNIQUE_KEY!" % missing_count) return False # If not, loop through the missing and create issues self.failure( "%s/%s (%d%%) of models lack a UNIQUE_KEY" % ( intcomma(missing_count), model_count, calculate.percentage(missing_count, model_count) ) )
def test_model_unique_key(self): """ Verify that each model has a UNIQUE_KEY attribute set. """ results = [] # Loop through the models for m in get_model_list(): # Verify the model has a UNIQUE_KEY attribute exists = m().UNIQUE_KEY is not None # Verify that the field names are not lowercase lowercases = sum([ 1 for k in m().get_unique_key_list() if re.search("[a-z]{1,}", k) ]) or 0 if lowercases: exists = False # Verify that the fields actually exist missing = [ f for f in m().get_unique_key_list() if not hasattr(m(), f.lower()) and not hasattr(m(), f.lower() + "_id") ] if missing: exists = False # Pass out results results.append([m().klass_group, m.__name__, exists]) # Print results self.attr_test_output("model", "UNIQUE_KEY", results)
def test_csv_gettrs(self): """ Verify that get_csv_name methods work for all models. """ for m in get_model_list(): self.assertEqual(m.objects.get_csv_name(), m().get_csv_name()) self.assertEqual(m.objects.get_csv_path(), m().get_csv_path())
def handle(self, *args, **kwargs): """ Make it happen. """ # Loop through all the models and find any fields without docs field_count = 0 missing_list = [] for m in get_model_list(): field_list = m().get_field_list() field_count += len(field_list) for f in field_list: if not self.has_docs(f): self.log("Missing: %s.%s.%s" % (m().klass_group, m().klass_name, f)) missing_list.append((m, f)) # If everything is done, declare victory if not missing_list: self.success("All %s fields documented!" % field_count) return False # If not, loop through the missing and create issues missing_count = len(missing_list) self.failure("%s/%s (%d%%) of fields lack documentation" % (intcomma(missing_count), intcomma(field_count), calculate.percentage(missing_count, field_count)))
def load(self): """ Loads the cleaned up csv files into the database """ if self.verbosity: self.header("Loading data files") model_list = [ x for x in get_model_list() if os.path.exists(x.objects.get_csv_path()) ] if self.resume_mode: # get finished load command logs of last update prev_loaded = [ x.file_name for x in self.log_record.called.filter( command='loadcalaccessrawfile', finish_datetime__isnull=False ) ] self.log("{} models already loaded.".format(len(prev_loaded))) # remove these from model_list model_list = [x for x in model_list if x._meta.db_table not in prev_loaded] if self.verbosity: model_list = progress.bar(model_list) for model in model_list: call_command( "loadcalaccessrawfile", model.__name__, verbosity=self.verbosity, keep_files=self.keep_files, app_name=self.app_name, )
def handle(self, *args, **kwargs): """ Make it happen. """ # Loop through all the models and find any fields without docs field_count = 0 missing_list = [] for m in get_model_list(): field_list = m().get_field_list() field_count += len(field_list) for f in field_list: if not self.has_docs(f): self.log("Missing: %s.%s.%s" % ( m().klass_group, m().klass_name, f ) ) missing_list.append((m, f)) # If everything is done, declare victory if not missing_list: self.success("All %s fields documented!" % field_count) return False # If not, loop through the missing and create issues missing_count = len(missing_list) self.failure( "%s/%s (%d%%) of fields lack documentation" % ( intcomma(missing_count), intcomma(field_count), calculate.percentage(missing_count, field_count) ) )
def handle(self, *args, **kwargs): self.header("Totaling CAL-ACCESS tables and rows") model_list = get_model_list() self.log(" %s total tables" % len(model_list)) record_count = 0 for m in model_list: record_count += m.objects.count() self.log(" %s total records" % intcomma(record_count))
def test_models(self): """ Make sure all the models have admins. """ model_list = [m.__name__ for m in get_model_list()] admin_list = [a.replace("Admin", "") for a in admin.__all__] missing = set(model_list).difference(admin_list) self.assertEqual(missing, set([]))
def _test_choices(self, field_name): """ Verify that proper choices appear for the provided field. """ message_list = [] # Loop through the models for m in get_model_list(): # And then through the fields for f in m._meta.fields: # Only test against the provided field name if not f.name == field_name: continue if not f.choices: message_list.append(( m.__name__, field_name, "Has no CHOICES defined" )) if not f.documentcloud_pages: message_list.append(( m.__name__, field_name, "Has no `documentcloud_pages_urls` defined" )) # Pull out all the choices in that field slug_list = [] for slug, name in f.choices: # Make sure that each has a definition if not name: message_list.append(( m.__name__, field_name, "Value '%s' undefined in CHOICES" % slug )) if name.lower() == 'unknown': message_list.append(( m.__name__, field_name, "Value '%s' defined as 'Unknown'" % slug )) slug_list.append(slug) # The query the database and make sure everything in # there has a matching definition in the choices for value, count in m.objects.values_list( field_name, ).annotate(Count(field_name)): message_list.append(( m.__name__, field_name, "Value '%s' in database but not in CHOICES" % value )) return message_list
def handle(self, *args, **kwargs): self.docs_dir = os.path.join( settings.REPO_DIR, 'docs' ) self.target_path = os.path.join(self.docs_dir, 'models.rst') model_list = sorted(get_model_list(), key=lambda x:x().klass_name) group_list = {} empty_files = [] for m in model_list: # add doc_id, page_url list to each model m.docs = {} for doc in m.DOCUMENTCLOUD_PAGES: doc_title = doc.get_doc_data()['title'] try: m.docs[doc_title].extend( zip( doc.get_page_urls(), doc.get_thumbnail_urls() ) ) except KeyError: m.docs[doc_title] = list(zip( doc.get_page_urls(), doc.get_thumbnail_urls() )) # add choice field list to model m.choice_fields = [] for field in m._meta.fields: if len(field.choices) > 0: # add doc_id, page_url list to each choice field field.docs = {} for doc in field.documentcloud_pages: doc_title = doc.get_doc_data()['title'] try: field.docs[doc_title].extend( zip( doc.get_page_urls(), doc.get_thumbnail_urls() ) ) except KeyError: field.docs[doc_title] = list(zip( doc.get_page_urls(), doc.get_thumbnail_urls() )) m.choice_fields.append(field) try: group_list[m().klass_group].append(m) except KeyError: group_list[m().klass_group] = [m] if m.objects.count() == 0: empty_files.append(m) group_list = sorted(group_list.items(), key=lambda x:x[0]) context = { 'group_list': group_list, 'model_count': len(model_list), 'empty_files': empty_files, } rendered = render_to_string('toolbox/models.rst', context) with open(self.target_path, 'w') as target_file: target_file.write(rendered)
def test_model_unique_key(self): """ Verify that each model has a UNIQUE_KEY attribute set. """ results = [] for m in get_model_list(): exists = m().UNIQUE_KEY is not None results.append([m.__name__, exists]) self.attr_test_output("model", "UNIQUE_KEY", results)
def load(self): """ Loads the cleaned up csv files into the database """ if self.verbosity: self.header("Loading data files") for model in get_model_list(): call_command("loadcalaccessrawfile", model.__name__)
def get_context_data(self, **kwargs): context = super(CalAccessFileList, self).get_context_data(**kwargs) model_list = get_model_list() context['model_list'] = model_list context['title'] = 'Raw files' context[ 'description'] = "Definitions, record layouts and data dictionaries for the {} raw \ files released from the California Secretary of State's CAL-ACCESS database. For experts only.".format( len(model_list)) return context
def get_models(self): """ Returns all the CAL-ACCESS models connected with this form. """ models = [] for model in get_model_list(): if self in [x[0] for x in model().get_filing_forms_w_sections()]: models.append(model) return models
def model(self): """ Returns the RawDataFile's corresponding CalAccess database model object. """ try: return [ m for m in get_model_list() if m().db_table == self.file_name ][0] except IndexError: return None
def handle(self, *args, **options): super(Command, self).handle(*args, **options) model_list = sorted( get_model_list(), key=lambda x: (x().klass_group, x().klass_name) ) results = [] for m in model_list: if self.verbosity > 1: self.log( " Verifying {0}.{1} fields".format( m().klass_group, m.__name__, ) ) for f in m._meta.fields: if f.choices: for value, count in m.objects.order_by().values_list( f.name, ).annotate(Count(f.name)): if ( value not in [x[0] for x in f.choices] and value != '' and value is not None ): if self.verbosity > 2: self.failure( " Undefined value for {0}: {1} ({2} occurrences)".format( f.name, value, count ) ) results.append(( m().klass_group, m.__name__, f.name, value, count, )) if len(results) > 0: self.failure("{} undefined choice field values".format(len(results))) table = agate.Table( results, ['group', 'model', 'field', 'undefined_value', 'occurrences'] ) table.print_table(max_rows=None, max_column_width=50) else: self.success("No undefined choice field values")
def test_model_str(self): """ Verify that __str__ methods exist and work for all models. """ results = [] for m in get_model_list(): # Test that works m().__str__() # Make sure that it is customized and not the Django defalult is_custom = m().__str__() != '%s object' % m.__name__ results.append([m.__name__, is_custom]) self.attr_test_output("model", "__str__", results)
def test_model_doc(self): """ Verify that __doc__ methods exist and work for all models. """ results = [] for m in get_model_list(): if m().__doc__.startswith(m.__name__): exists = False else: exists = True results.append([m.__name__, exists]) self.attr_test_output("model", "__doc__", results)
def test_model_documentcloud_pages(self): """ Verify that each model has DOCUMENTCLOUD_PAGES defined """ results = [] for m in get_model_list(): if m().DOCUMENTCLOUD_PAGES: exists = True else: exists = False results.append([m().klass_group, m.__name__, exists]) self.attr_test_output("model", "DOCUMENTCLOUD_PAGES", results)
def test_choices(self): """ Verify that valid choices are available for all expected fields on all models. """ # substrings that appear in choice fields choice_field_strs = [ '_cd', '_code', '_type', 'status', '_lvl', 'reportname', 'form_id', ] exceptions = [ 'LookupCodesCd.code_type', 'S497Cd.sup_off_cd', 'FilerStatusTypesCd.status_type', 'FilerStatusTypesCd.status_desc', 'FilerTypesCd.filer_type', ] results = [] model_list = sorted(get_model_list(), key=lambda x: (x().klass_group, x().klass_name)) for m in model_list: for f in m._meta.fields: if (any(x in f.name for x in choice_field_strs) and f.name != 'memo_code' and f.__class__ is not ForeignKeyField and '{}.{}'.format(m().klass_name, f.name) not in exceptions): if not f.choices: results.append((m().klass_group, m.__name__, f.name, "Has no CHOICES defined")) if not f.documentcloud_pages: results.append( (m().klass_group, m.__name__, f.name, "Has no `documentcloud_pages` defined")) # Pull out all the choices in that field for slug, name in f.choices: # Make sure that each has a definition if not name or name == '': results.append( (m().klass_group, m.__name__, f.name, "Value '%s' undefined in CHOICES" % slug)) table = agate.Table(results, ['group', 'model', 'field', 'message']) table.print_table(max_rows=None, max_column_width=50)
def test_field_docs(self): """ Verify that all fields have verbose_name or help_text documentation. """ for m in get_model_list(): for f in m().get_field_list(): if f.name == "id": continue if f.help_text: continue if f.__dict__["_verbose_name"]: continue warnings.warn("%s.%s field undocumented" % (m().klass_name, f.name))
def handle(self, *args, **options): super(Command, self).handle(*args, **options) """ Connect to Github using token stored in environment, loop over model fields, and \ create an issue for any choice field missing """ self.dry_run = options["dry_run"] # set up connect to Github account self.gh = Github(os.getenv('GITHUB_TOKEN')) self.org = self.gh.get_organization("california-civic-data-coalition") self.repo = self.org.get_repo("django-calaccess-raw-data") self.labels = [ self.repo.get_label("small"), self.repo.get_label("documentation"), self.repo.get_label("enhancement"), ] self.header( "Creating GitHub issues for model choice fields" ) choice_field_strs = [ '_cd', '_code', 'type', 'status', '_lvl', 'reportname', 'form_id', ] excluded_fields = [ 'LookupCodesCd.code_type', 'S497Cd.sup_off_cd', ] model_list = sorted( get_model_list(), key=lambda x: (x().klass_group, x().klass_name) ) for m in model_list: for f in m._meta.fields: if ( any(x in f.name for x in choice_field_strs) and f.name != 'memo_code' and f.__class__ is not models.ForeignKey and '{}.{}'.format(m().klass_name, f.name) not in excluded_fields ): # make an issue for every choice field missing docs # includes those that are also missing choices if not f.documentcloud_pages: self.create_issue(f)
def load(self): """ Loads the cleaned up csv files into the database """ if self.verbosity: self.header("Loading data files") model_list = get_model_list() for model in progress.bar(model_list): call_command( "loadcalaccessrawfile", model.__name__, verbosity=self.verbosity, )
def test_filing_forms(self): """ Verify that each model with a form_type or form_id field has FILING_FORMS defined """ results = [] for m in get_model_list(): field_names = m._meta.get_all_field_names() if 'form_type' in field_names or 'form_id' in field_names: if m().FILING_FORMS: exists = True else: exists = False results.append([m().klass_group, m.__name__, exists]) self.attr_test_output("model", "FILING_FORMS", results)
def test_field_docs(self): """ Verify that all fields have verbose_name or help_text documentation. """ for m in get_model_list(): for f in m().get_field_list(): if f.name == 'id': continue if f.help_text: continue if f.__dict__['_verbose_name']: continue warnings.warn("%s.%s field undocumented" % (m().klass_name, f.name))
def handle(self, *args, **kwargs): self.docs_dir = os.path.join(settings.REPO_DIR, 'docs') self.target_path = os.path.join(self.docs_dir, 'models.rst') model_list = sorted(get_model_list(), key=lambda x: x().klass_name) group_list = {} empty_files = [] for m in model_list: # add doc_id, page_url list to each model m.docs = {} for doc in m.DOCUMENTCLOUD_PAGES: doc_title = doc.get_doc_data()['title'] try: m.docs[doc_title].extend( zip(doc.get_page_urls(), doc.get_thumbnail_urls())) except KeyError: m.docs[doc_title] = list( zip(doc.get_page_urls(), doc.get_thumbnail_urls())) # add choice field list to model m.choice_fields = [] for field in m._meta.fields: if len(field.choices) > 0: # add doc_id, page_url list to each choice field field.docs = {} for doc in field.documentcloud_pages: doc_title = doc.get_doc_data()['title'] try: field.docs[doc_title].extend( zip(doc.get_page_urls(), doc.get_thumbnail_urls())) except KeyError: field.docs[doc_title] = list( zip(doc.get_page_urls(), doc.get_thumbnail_urls())) m.choice_fields.append(field) try: group_list[m().klass_group].append(m) except KeyError: group_list[m().klass_group] = [m] if m.objects.count() == 0: empty_files.append(m) group_list = sorted(group_list.items(), key=lambda x: x[0]) context = { 'group_list': group_list, 'model_count': len(model_list), 'empty_files': empty_files, } rendered = render_to_string('toolbox/models.rst', context) with open(self.target_path, 'w') as target_file: target_file.write(rendered)
def test_field_help_text(self): """ Verify that all fields have help_text documentation. """ results = [] for m in get_model_list(): for f in m().get_field_list(): if f.name == 'id': continue if f.help_text: exists = True else: exists = False results.append(("%s.%s" % (m.__name__, f.name), exists)) self.attr_test_output("field", "help_text", results)
def handle(self, *args, **options): docs = set() cached = set() for m in get_model_list(): docs.update(set(m().DOCUMENTCLOUD_PAGES)) for form in all_filing_forms: docs.add(form.documentcloud) for doc in docs: if doc and doc.metadata_filename not in cached: logger.debug('Caching {}'.format(doc.metadata_filename)) doc._cache_metadata() cached.add(doc.metadata_filename)
def test_field_verbose_name(self): """ Verify that all fields have verbose_name documentation. """ results = [] for m in get_model_list(): for f in m().get_field_list(): if f.name == 'id': continue if f.__dict__['_verbose_name']: exists = True else: exists = False results.append(("%s.%s" % (m.__name__, f.name), exists)) self.attr_test_output("field", "verbose_name", results)
def get_context_data(self): model_list = calaccess_raw.get_model_list() form_list = FORMS object_list = [ dict( name='Processed files from the California Civic Data Coalition', description="Definitions, record layouts and data dictionaries for the \ processed data files released by the California Civic Data Coalition. Recommended for beginners and regular use.", url=reverse("ccdc_file_list"), ), dict( name='Raw files from CAL-ACCESS', description="Definitions, record layouts and data dictionaries for the {} raw \ files released from the California Secretary of State's CAL-ACCESS database. For experts only.".format(len(model_list)), url=reverse("calaccess_file_list"), ), dict( name='Official CAL-ACCESS forms', description="Descriptions, samples and other documentation for \ the {} forms that campaigns and lobbyists use to disclose activity to California's Secretary of \ State.".format(len(form_list)), url=reverse("form_list"), ), dict( name='Official CAL-ACCESS documentation', description="The jumbled, fragmentary and unreliable documentation \ for the CAL-ACCESS database provided by California's Secretary of State. For more authoritative \ information refer to the materials above.", url=reverse("official_documentation"), ), dict( name='Technical documentation', description="A user manual for the collection of \ California Civic Data Coalition applications that power this site. For developers only.", url="//django-calaccess.californiacivicdata.org" ), dict( name="Frequently asked questions", description="Answers to common questions about this project and the underlying CAL-ACCESS database.", url=reverse("faq"), ), ] return { 'object_list': object_list, 'title': "Documentation", 'description': "How to work with our data tracking campaign finance and lobbying activity in \ California politics." }
def load(self): """ Loads the cleaned up csv files into the database """ if self.verbosity: self.header("Loading data files") model_list = get_model_list() if self.verbosity: model_list = progress.bar(model_list) for model in model_list: call_command( "loadcalaccessrawfile", model.__name__, verbosity=self.verbosity, )
def _test_choices(self, field_name): """ Verify that proper choices appear for the provided field. """ message_list = [] # Loop through the models for m in get_model_list(): # And then through the fields for f in m._meta.fields: # Only test against the provided field name if not f.name == field_name: continue if not f.choices: message_list.append((m().klass_group, m.__name__, field_name, "Has no CHOICES defined")) if not f.documentcloud_pages: message_list.append( (m().klass_group, m.__name__, field_name, "Has no `documentcloud_pages` defined")) # Pull out all the choices in that field slug_list = [] for slug, name in f.choices: # Make sure that each has a definition if not name: message_list.append( (m().klass_group, m.__name__, field_name, "Value '%s' undefined in CHOICES" % slug)) if name.lower() == 'unknown': message_list.append( (m().klass_group, m.__name__, field_name, "Value '%s' defined as 'Unknown'" % slug)) slug_list.append(slug) # The query the database and make sure everything in # there has a matching definition in the choices for value, count in m.objects.values_list( field_name, ).annotate(Count(field_name)): message_list.append( (m().klass_group, m.__name__, field_name, "Value '%s' in database but not in CHOICES" % value)) return message_list
def handle(self, *args, **kwargs): self.docs_dir = os.path.join(settings.REPO_DIR, "docs") self.target_path = os.path.join(self.docs_dir, "models.rst") model_list = sorted(get_model_list(), key=lambda x: x().klass_name) group_list = {} empty_files = [] for m in model_list: try: group_list[m().klass_group].append(m) except KeyError: group_list[m().klass_group] = [m] if m.objects.count() == 0: empty_files.append(m) group_list = sorted(group_list.items(), key=lambda x: x[0]) context = {"group_list": group_list, "model_count": len(model_list), "empty_files": empty_files} rendered = render_to_string("toolbox/models.rst", context) with open(self.target_path, "w") as target_file: target_file.write(rendered)