def testModel(self, test_sampler):
"""
:param test_sampler:
:return: TesterResponse
"""
real_target_all = []
predicted_target_all = []
self.eval() # toggle eval
for batch_number, batch in enumerate(test_sampler):
for permutation in self.col_permutations:
batch.blank_columns = permutation
#batch.blank_columns = []
logging.debug(
'[EPOCH-BATCH] testing batch: {batch_number}'.format(
batch_number=batch_number))
# get real and predicted values by running the model with the input of this batch
predicted_target = self.forward(
batch.getInput(flatten=self.flatInput))
real_target = batch.getTarget(flatten=self.flatTarget)
# append to all targets and all real values
real_target_all += real_target.data.tolist()
predicted_target_all += predicted_target.data.tolist()
if batch is None:
logging.error('there is no data in test, we should not be here')
return
# caluclate the error for all values
predicted_targets = batch.deflatTarget(np.array(predicted_target_all))
real_targets = batch.deflatTarget(np.array(real_target_all))
r_values = {}
# calculate r and other statistical properties of error
for target_key in real_targets:
r_values[target_key] = explained_variance_score(
real_targets[target_key],
predicted_targets[target_key],
multioutput='variance_weighted')
# calculate error using error function
errors = {
target_key: float(
self.errorFunction(
Variable(torch.FloatTensor(predicted_targets[target_key])),
Variable(torch.FloatTensor(
real_targets[target_key]))).data[0])
for target_key in real_targets
}
error = np.average([errors[key] for key in errors])
r_value = np.average([r_values[key] for key in r_values])
resp = TesterResponse(error=error,
accuracy=r_value,
predicted_targets=predicted_targets,
real_targets=real_targets)
return resp
def forward(self, input):
"""
This is what is called when the model is forwarded
:param input:
:return:
"""
logging.error('You must define a forward method for this model')
pass
def setup(self, sample_batch, **kwargs):
"""
this is what is called when the model object is instantiated
:param sample_batch:
:param use_cuda:
:return:
"""
logging.error('You must define a setup method for this model')
pass
def __init__(self):
self._mongo = TinyMongoClient(CONFIG.LOCALSTORE_PATH)
try:
self._collection = self._mongo.mindsdb[self._entity_name]
except:
logging.error(
'No collection will be found, db corrupted, truncating it')
shutil.rmtree(CONFIG.LOCALSTORE_PATH)
raise ValueError(
'MindsDB local document store corruped. No other way to put this, trained model data will be lost'
)
self.setup()
def getDS(from_data):
'''
Get a datasource give the input
:param input: a string or an object
:return: a datasource
'''
if isinstance(from_data, DataSource):
from_ds = from_data
elif Path(from_data).is_file():
from_ds = CSVFileDS(from_data)
else: # assume is a query
logging.error('No data matched the input data')
return from_ds
def getDS(from_data):
'''
Get a datasource give the input
:param input: a string or an object
:return: a datasource
'''
if isinstance(from_data, DataSource):
from_ds = from_data
else: # assume is a file
from_ds = FileDS(from_data)
if from_ds is None:
logging.error('No data matched the input data')
return from_ds
def testModel(self, test_sampler):
"""
:param test_sampler:
:return: TesterResponse
"""
real_target_all = []
predicted_target_all = []
real_target_all_ret = []
predicted_target_all_ret = []
self.eval() # toggle eval
perm_index = 0
for batch_number, batch in enumerate(test_sampler):
# do only one permutation at a time, if we 2 or more columns
if len(self.input_column_names) > 1:
perms = [self.col_permutations[perm_index], []]
perm_index = perm_index + 1 if perm_index + 1 < len(
self.col_permutations) else 0
else:
perms = [[]]
for permutation in perms:
batch.blank_columns = permutation
#batch.blank_columns = []
logging.debug(
'[EPOCH-BATCH] testing batch: {batch_number}'.format(
batch_number=batch_number))
# get real and predicted values by running the model with the input of this batch
predicted_target = self.forwardWrapper(batch)
real_target = batch.getTarget(flatten=self.flatTarget)
# append to all targets and all real values
real_target_all += real_target.data.tolist()
predicted_target_all += predicted_target.data.tolist()
if len(permutation) == 0:
# append to all targets and all real values
real_target_all_ret += real_target.data.tolist()
predicted_target_all_ret += predicted_target.data.tolist()
if batch is None:
logging.error('there is no data in test, we should not be here')
return
# caluclate the error for all values
predicted_targets = batch.deflatTarget(np.array(predicted_target_all))
real_targets = batch.deflatTarget(np.array(real_target_all))
# caluclate the error for all values
predicted_targets_ret = batch.deflatTarget(
np.array(predicted_target_all_ret))
real_targets_ret = batch.deflatTarget(np.array(real_target_all_ret))
r_values = {}
# calculate r and other statistical properties of error
for target_key in real_targets_ret:
r_values[target_key] = explained_variance_score(
real_targets_ret[target_key],
predicted_targets_ret[target_key],
multioutput='variance_weighted')
# calculate error using error function
errors = {
target_key: float(
self.errorFunction(
Variable(torch.FloatTensor(predicted_targets[target_key])),
Variable(torch.FloatTensor(
real_targets[target_key]))).item())
for target_key in real_targets
}
error = np.average([errors[key] for key in errors])
r_value = np.average([r_values[key] for key in r_values])
resp = TesterResponse(error=error,
accuracy=r_value,
predicted_targets=predicted_targets_ret,
real_targets=real_targets_ret)
self.current_accuracy = r_value
return resp