def test_wrapping(self):
''' Verifica se retorna o dataset apenas com o wrapping '''
model = BaseModel()
options = {
**{
"categorias": [
'nm_indicador', 'nu_competencia', 'vl_indicador', 'lat_mun', 'long_mun'
],
"pivot": None
}, **COMMON_OPTIONS
}
result = "".join(model.wrap_result(SAMPLE_DATAFRAME.copy(), options).split())
str_expected = COMMON_EXPECTED_RESPONSE_STRING.format(
"""
"nm_indicador": "Ficticio",
"nu_competencia": 2099,
"vl_indicador": 1.0
},
{
"nm_indicador": "Ficticio",
"nu_competencia": 2047,
"vl_indicador": 0.5
"""
)
expected = "".join(str_expected.split())
self.assertEqual(result, expected)
def test_set_cache(self):
"""Test BaseModel.set_cache"""
self.assertIsInstance(BaseModel._cache, FileSystemCache)
old_cache = BaseModel._cache
BaseModel.set_cache(NullCache())
self.assertIsInstance(BaseModel._cache, NullCache)
SampleModel.set_cache(old_cache)
self.assertIsInstance(BaseModel._cache, NullCache)
self.assertIsInstance(SampleModel._cache, FileSystemCache)
def test_set_cache(self):
"""Test BaseModel.set_cache"""
self.assertIsInstance(BaseModel._cache, FileSystemCache)
old_cache = BaseModel._cache
BaseModel.set_cache(NullCache())
self.assertIsInstance(BaseModel._cache, NullCache)
SampleModel.set_cache(old_cache)
self.assertIsInstance(BaseModel._cache, NullCache)
self.assertIsInstance(SampleModel._cache, FileSystemCache)
def setUp(self):
"""BaseModel test set up"""
if os.path.isfile("/tmp/box.db"):
os.unlink("/tmp/box.db")
DBHelper().set_db("/tmp/box.db")
InstallHelper.reset()
cache = FileSystemCache("/tmp/werkzeug")
cache.clear()
BaseModel.set_cache(cache)
SampleModel.install()
def setUp(self):
"""BaseModel test set up"""
if os.path.isfile('/tmp/box.db'):
os.unlink('/tmp/box.db')
DBHelper().set_db('/tmp/box.db')
InstallHelper.reset()
cache = FileSystemCache('/tmp/werkzeug')
cache.clear()
BaseModel.set_cache(cache)
SampleModel.install()
def test_multiple_aggr(self):
''' Verifica se retorna np.mean se agregação nula '''
model = BaseModel()
aggrs = ['sum', 'count', 'xpto', '']
vals = ['vl_indicador', 'cd_municipio', 'nu_competencia',
'another_field']
expected = {
'vl_indicador': np.sum,
'cd_municipio': np.sum,
'nu_competencia': np.mean,
'another_field': np.mean
}
self.assertEqual(model.convert_aggr_to_np(aggrs, vals), expected)
def test_no_def(self):
''' Verifica se retorna np.mean se agregação nula '''
model = BaseModel()
self.assertRaises(
NotImplementedError,
model.get_repo
)
class Document(BaseModel):
__tablename__ = 'documents'
content = db.Column('content', db.Text(), nullable=False)
title = db.Column('title', db.String(200), nullable=False)
description = db.Column('description', db.Text(), nullable=True)
author = db.Column('author', db.String(200), nullable=True)
# Table metadata can be specified as follows -
__table_args__ = (
db.UniqueConstraint('title', 'is_deleted'),
db.Index(BaseModel.create_index(__tablename__, 'title', 'is_deleted'),
'title', 'is_deleted'),
)
def __str__(self):
return self.title
def __repr__(self):
return json.dumps(self.to_dict())
def to_dict(self, *args, **kwargs):
return {
'id': self.id,
'author': self.author,
'title': self.title,
'description': self.description,
'content': self.content
}
def test_get_collection_from_invalid_type(self):
''' Test if the method returns None if an invalid type is passed '''
self.assertEqual(
BaseModel.get_collection_from_type(
SAMPLE_DATAFRAME.copy(),
'invalid'
),
None
)
def test_resort_dataset_desc(self):
''' Test if dataset is ordered descending if all args are descending '''
self.assertEqual(
BaseModel.resort_dataset(
SAMPLE_DATAFRAME,
["-col_1", "-col_2", "-col_3"]
).to_dict('split')['data'],
[['d', 3, 3], ['c', 0, 0], ['b', 2, 2], ['a', 1, 1]]
)
def test_get_collection_from_type_first_occurence(self):
''' Test if the method returns the first item '''
self.assertEqual(
BaseModel.get_collection_from_type(
SAMPLE_DATAFRAME.copy(),
"first_occurence"
).to_dict(),
{"index": 0, "col_1": "d", "col_2": 3, "col_3": 3}
)
def test_resort_dataset_no_rule(self):
''' Test if dataset resturns as is if no order rule is sent '''
self.assertEqual(
BaseModel.resort_dataset(
SAMPLE_DATAFRAME,
["col_1", "col_2"]
).to_dict('split')['data'],
[['a', 1, 1], ['b', 2, 2], ['c', 0, 0], ['d', 3, 3]]
)
def test_get_collection_from_missing_type(self):
''' Test if the method returns None if no type is passed '''
self.assertEqual(
BaseModel.get_collection_from_type(
SAMPLE_DATAFRAME.copy(),
None
),
None
)
def test_get_collection_from_type_max(self):
''' Test if the method returns the item with maximum value in colum '''
self.assertEqual(
BaseModel.get_collection_from_type(
SAMPLE_DATAFRAME.copy(),
"max",
"col_2"
).to_dict(),
{"col_1": "d", "col_2": 3, "col_3": 3}
)
def test_filter_dataset_nofilter(self):
''' Test if dataset returns when no filter is actually sent '''
self.assertEqual(
BaseModel.filter_pandas_dataset(SAMPLE_DATAFRAME, None).to_dict(),
{
'col_1': {0: 'd', 1: 'b', 2: 'a', 3: 'c'},
'col_2': {0: 3, 1: 2, 2: 1, 3: 0},
'col_3': {0: 3, 1: 2, 2: 1, 3: 0}
}
)
def test_get_collection_from_type_from_id(self):
''' Test if the method returns the item with the passed numeric id '''
self.assertEqual(
BaseModel.get_collection_from_type(
SAMPLE_DATAFRAME.copy(),
"from_id",
"col_2",
2
).to_dict(),
{"col_1": "b", "col_2": 2, "col_3": 2}
)
def test_reform_filters_full(self):
''' Test if filters are classified correctly as pre and post '''
self.assertEqual(
BaseModel.reform_filters_for_pandas(
['post-eq-a-1', 'and', 'in-b-2-3', 'and', 'post-nn-c', 'or', 'eq-d-4']
),
(
['in-b-2-3', 'and', 'eq-d-4'],
[['post', 'eq', 'a', '1'], 'and', ['post', 'nn', 'c']]
)
)
def test_filter_dataset_nn(self):
''' Test if dataset is filtered by NN clause '''
self.assertEqual(
BaseModel.filter_pandas_dataset(
SAMPLE_DATAFRAME_NA,
[['post', 'nn', 'col_2']]
).to_dict(),
{
'col_1': {0: 'd', 1: 'b', 2: 'a'},
'col_2': {0: 3, 1: 2, 2: 1},
'col_3': {0: 3, 1: 2, 2: 1}
}
)
def test_filter_dataset_mixed(self):
''' Test if dataset is filtered by multiple clauses '''
self.assertEqual(
BaseModel.filter_pandas_dataset(
SAMPLE_DATAFRAME_NA,
[['post', 'in', 'col_3', '3', '1'], ['post', 'in', 'col_2', '3', '2']]
).to_dict(),
{
'col_1': {0: 'd'},
'col_2': {0: 3},
'col_3': {0: 3}
}
)
def test_filter_dataset_in(self):
''' Test if dataset is filtered by IN clause '''
self.assertEqual(
BaseModel.filter_pandas_dataset(
SAMPLE_DATAFRAME,
[['post', 'in', 'col_2', '3', '2']]
).to_dict(),
{
'col_1': {0: 'd', 1: 'b'},
'col_2': {0: 3, 1: 2},
'col_3': {0: 3, 1: 2}
}
)
def test_filter_dataset_eq(self):
''' Test if dataset is filtered by EQ clause '''
self.assertEqual(
BaseModel.filter_pandas_dataset(
SAMPLE_DATAFRAME,
[['post', 'eq', 'col_2', '3']]
).to_dict(),
{
'col_1': {0: 'd'},
'col_2': {0: 3},
'col_3': {0: 3}
}
)
def test_build_derivatives(self):
''' Test if derivate object is added to the collection '''
options = {'cd_analysis_unit': 2}
rules = {
"instances": [{"name": "inst_1", 'type': 'max', 'named_prop': 'col_2'}]
}
sources = {"dataset": SAMPLE_DATAFRAME.copy()}
(der_data, der_anynodata) = BaseModel.build_derivatives(
rules,
options,
sources,
{}
)
self.assertEqual(
der_data["inst_1"].to_dict(),
{"col_1": "d", "col_2": 3, "col_3": 3}
)
self.assertEqual(der_anynodata, False)
def test_build_derivatives_nodata(self):
''' Test the derivate objects is added with no_data flag '''
options = {'cd_analysis_unit': 99}
rules = {
"instances": [{"name": "inst_1", 'type': 'from_id', 'named_prop': 'col_2'}]
}
sources = {"dataset": SAMPLE_DATAFRAME.copy()}
(der_data, der_anynodata) = BaseModel.build_derivatives(
rules,
options,
sources,
{}
)
self.assertEqual(
der_data["inst_1"],
None
)
self.assertEqual(der_anynodata, True)
model_save_dir = opt['save_dir'] + '/' + model_id
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
helper.save_config(opt, model_save_dir + '/config.json', verbose=True)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
model_save_dir + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\tdev_f1\ttest_loss\ttest_f1")
# print model info
helper.print_config(opt)
# model
base_model = BaseModel(opt=opt, emb_matrix=emb_matrix)
id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
dev_f1_history = []
current_lr = opt['lr']
global_step = 0
global_start_time = time.time()
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
max_steps = len(train_batch) * opt['num_epoch']
# start training
max_dev_f1, max_dev_id = 0, 0
max_test_f1, max_test_id = 0, 0
for epoch in range(1, opt['num_epoch'] + 1):
train_loss = 0
def test_default_on_unmapped(self):
''' Verifica se retorna np.mean se agregação não prevista '''
self.assertEqual(BaseModel.aggr_to_np('xpto'), np.mean)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
model_save_dir + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\tdev_f1\ttest_loss\ttest_f1")
# print model info
helper.print_config(opt)
# model
base_outputs = ['placeholder' for _ in range(0, len(train_batch))]
if opt['base_mode'] == 3:
base_outputs = []
base_model_file = opt['save_dir'] + '/' + opt['base_id'] + '/best_model.pt'
print("Loading base model from {}".format(base_model_file))
base_opt = torch_utils.load_config(base_model_file)
base_model = BaseModel(opt=base_opt)
base_model.load(base_model_file)
base_model.model.eval()
for _, batch in enumerate(train_batch):
inputs = [b.cuda() for b in batch[:10]
] if opt['cuda'] else [b for b in batch[:10]]
base_logits, _, _ = base_model.model(inputs)
base_outputs.append([base_logits.data.cpu().numpy()])
teacher_outputs = []
if opt['use_teacher']:
teacher_model_file = opt['save_dir'] + '/' + opt[
'teacher_id'] + '/best_model.pt'
print("Loading teacher model from {}".format(teacher_model_file))
teacher_opt = torch_utils.load_config(teacher_model_file)
teacher_model = TeacherModel(opt=teacher_opt)
def test_reform_filters_no_filter(self):
''' Test if None is returned if no filter is sent to the method '''
self.assertEqual(
BaseModel.reform_filters_for_pandas(None),
(None, None)
)
def test_sum(self):
''' Verifica se retorna np.sum '''
self.assertEqual(BaseModel.aggr_to_np('sum'), np.sum)
def test_default_on_none(self):
''' Verifica se retorna np.mean se agregação nula '''
self.assertEqual(BaseModel.aggr_to_np(None), np.mean)
def test_sum_on_count(self):
''' Verifica se não retorna nada e a agregação for count '''
self.assertEqual(BaseModel.aggr_to_np('count'), np.sum)
def test_default_on_empty(self):
''' Verifica se retorna np.mean se agragação vazia '''
self.assertEqual(BaseModel.aggr_to_np(''), np.mean)
def test_single_aggr(self):
''' Verifica se retorna np.mean se agregação nula '''
model = BaseModel()
aggrs = 'sum'
self.assertEqual(model.convert_aggr_to_np(aggrs, None), np.sum)