def load_data(self, load_towns=None, randomize=True, exclude_fields=()):
data_records = []
for town in RelocatedTown.objects.all():
if load_towns and town.name not in load_towns:
continue
processing_log.info("Loading town {0:s}".format(town.name))
new_data = conversion.features_to_dicts(town.parcels.layer, chosen_field=self.chosen_field)
data_records += new_data
self._set_up_loaded_data(data_records,randomize=randomize)
def load_data(self, load_towns=None, randomize=True, exclude_fields=()):
data_records = []
for town in RelocatedTown.objects.all():
if load_towns and town.name not in load_towns:
continue
processing_log.info("Loading town {0:s}".format(town.name))
new_data = conversion.features_to_dicts(town.parcels.layer, chosen_field=self.chosen_field)
data_records += new_data
self._set_up_loaded_data(data_records, randomize=randomize)
def check_town_influence(self, num_iterations=5):
data_records = {}
for town in RelocatedTown.objects.all():
processing_log.info("Loading town {0:s}".format(town.name))
new_data = conversion.features_to_dicts(town.parcels.layer, chosen_field=self.chosen_field)
data_records[town.name] = new_data
towns = list(data_records.keys())
for exclude_town in towns:
current_records = []
# compose a dataset of all towns but the loaded ones
for town in towns:
if town == exclude_town:
processing_log.info("Excluding town {} for run".format(exclude_town))
continue
current_records += data_records[town]
# set it up on the object
self._set_up_loaded_data(data_records=current_records, randomize=True)
percent_incorrects = []
# run a number of iterations with the model
for iteration in range(num_iterations):
processing_log.debug("iteration {}".format(iteration))
self.reshuffle()
self.withhold_and_fit_model(withhold=0.1)
total_y_trues = np.count_nonzero(self.withheld_truth)
self.validate()
percent_incorrects.append(self.underpredict / total_y_trues)
processing_log.info(
"Mean percent incorrect (for true) when excluding {} = {}".format(
exclude_town, mean(percent_incorrects)
)
)
processing_log.info(
"Min percent incorrect (for true) when excluding {} = {}".format(exclude_town, min(percent_incorrects))
)
processing_log.info(
"Max percent incorrect (for true) when excluding {} = {}".format(exclude_town, max(percent_incorrects))
)
def check_town_influence(self, num_iterations=5):
data_records = {}
for town in RelocatedTown.objects.all():
processing_log.info("Loading town {0:s}".format(town.name))
new_data = conversion.features_to_dicts(town.parcels.layer, chosen_field=self.chosen_field)
data_records[town.name] = new_data
towns = list(data_records.keys())
for exclude_town in towns:
current_records = []
# compose a dataset of all towns but the loaded ones
for town in towns:
if town == exclude_town:
processing_log.info("Excluding town {} for run".format(exclude_town))
continue
current_records += data_records[town]
# set it up on the object
self._set_up_loaded_data(data_records=current_records, randomize=True)
percent_incorrects = []
# run a number of iterations with the model
for iteration in range(num_iterations):
processing_log.debug("iteration {}".format(iteration))
self.reshuffle()
self.withhold_and_fit_model(withhold=.1)
total_y_trues = np.count_nonzero(self.withheld_truth)
self.validate()
percent_incorrects.append(self.underpredict / total_y_trues)
processing_log.info("Mean percent incorrect (for true) when excluding {} = {}".format(exclude_town,mean(percent_incorrects)))
processing_log.info("Min percent incorrect (for true) when excluding {} = {}".format(exclude_town,min(percent_incorrects)))
processing_log.info("Max percent incorrect (for true) when excluding {} = {}".format(exclude_town,max(percent_incorrects)))
