def test_wrapping(self): ''' Verifica se retorna o dataset apenas com o wrapping ''' model = BaseModel() options = { **{ "categorias": [ 'nm_indicador', 'nu_competencia', 'vl_indicador', 'lat_mun', 'long_mun' ], "pivot": None }, **COMMON_OPTIONS } result = "".join(model.wrap_result(SAMPLE_DATAFRAME.copy(), options).split()) str_expected = COMMON_EXPECTED_RESPONSE_STRING.format( """ "nm_indicador": "Ficticio", "nu_competencia": 2099, "vl_indicador": 1.0 }, { "nm_indicador": "Ficticio", "nu_competencia": 2047, "vl_indicador": 0.5 """ ) expected = "".join(str_expected.split()) self.assertEqual(result, expected)
def test_set_cache(self): """Test BaseModel.set_cache""" self.assertIsInstance(BaseModel._cache, FileSystemCache) old_cache = BaseModel._cache BaseModel.set_cache(NullCache()) self.assertIsInstance(BaseModel._cache, NullCache) SampleModel.set_cache(old_cache) self.assertIsInstance(BaseModel._cache, NullCache) self.assertIsInstance(SampleModel._cache, FileSystemCache)
def setUp(self): """BaseModel test set up""" if os.path.isfile("/tmp/box.db"): os.unlink("/tmp/box.db") DBHelper().set_db("/tmp/box.db") InstallHelper.reset() cache = FileSystemCache("/tmp/werkzeug") cache.clear() BaseModel.set_cache(cache) SampleModel.install()
def setUp(self): """BaseModel test set up""" if os.path.isfile('/tmp/box.db'): os.unlink('/tmp/box.db') DBHelper().set_db('/tmp/box.db') InstallHelper.reset() cache = FileSystemCache('/tmp/werkzeug') cache.clear() BaseModel.set_cache(cache) SampleModel.install()
def test_multiple_aggr(self): ''' Verifica se retorna np.mean se agregação nula ''' model = BaseModel() aggrs = ['sum', 'count', 'xpto', ''] vals = ['vl_indicador', 'cd_municipio', 'nu_competencia', 'another_field'] expected = { 'vl_indicador': np.sum, 'cd_municipio': np.sum, 'nu_competencia': np.mean, 'another_field': np.mean } self.assertEqual(model.convert_aggr_to_np(aggrs, vals), expected)
def test_no_def(self): ''' Verifica se retorna np.mean se agregação nula ''' model = BaseModel() self.assertRaises( NotImplementedError, model.get_repo )
class Document(BaseModel): __tablename__ = 'documents' content = db.Column('content', db.Text(), nullable=False) title = db.Column('title', db.String(200), nullable=False) description = db.Column('description', db.Text(), nullable=True) author = db.Column('author', db.String(200), nullable=True) # Table metadata can be specified as follows - __table_args__ = ( db.UniqueConstraint('title', 'is_deleted'), db.Index(BaseModel.create_index(__tablename__, 'title', 'is_deleted'), 'title', 'is_deleted'), ) def __str__(self): return self.title def __repr__(self): return json.dumps(self.to_dict()) def to_dict(self, *args, **kwargs): return { 'id': self.id, 'author': self.author, 'title': self.title, 'description': self.description, 'content': self.content }
def test_get_collection_from_invalid_type(self): ''' Test if the method returns None if an invalid type is passed ''' self.assertEqual( BaseModel.get_collection_from_type( SAMPLE_DATAFRAME.copy(), 'invalid' ), None )
def test_resort_dataset_desc(self): ''' Test if dataset is ordered descending if all args are descending ''' self.assertEqual( BaseModel.resort_dataset( SAMPLE_DATAFRAME, ["-col_1", "-col_2", "-col_3"] ).to_dict('split')['data'], [['d', 3, 3], ['c', 0, 0], ['b', 2, 2], ['a', 1, 1]] )
def test_get_collection_from_type_first_occurence(self): ''' Test if the method returns the first item ''' self.assertEqual( BaseModel.get_collection_from_type( SAMPLE_DATAFRAME.copy(), "first_occurence" ).to_dict(), {"index": 0, "col_1": "d", "col_2": 3, "col_3": 3} )
def test_resort_dataset_no_rule(self): ''' Test if dataset resturns as is if no order rule is sent ''' self.assertEqual( BaseModel.resort_dataset( SAMPLE_DATAFRAME, ["col_1", "col_2"] ).to_dict('split')['data'], [['a', 1, 1], ['b', 2, 2], ['c', 0, 0], ['d', 3, 3]] )
def test_get_collection_from_missing_type(self): ''' Test if the method returns None if no type is passed ''' self.assertEqual( BaseModel.get_collection_from_type( SAMPLE_DATAFRAME.copy(), None ), None )
def test_get_collection_from_type_max(self): ''' Test if the method returns the item with maximum value in colum ''' self.assertEqual( BaseModel.get_collection_from_type( SAMPLE_DATAFRAME.copy(), "max", "col_2" ).to_dict(), {"col_1": "d", "col_2": 3, "col_3": 3} )
def test_filter_dataset_nofilter(self): ''' Test if dataset returns when no filter is actually sent ''' self.assertEqual( BaseModel.filter_pandas_dataset(SAMPLE_DATAFRAME, None).to_dict(), { 'col_1': {0: 'd', 1: 'b', 2: 'a', 3: 'c'}, 'col_2': {0: 3, 1: 2, 2: 1, 3: 0}, 'col_3': {0: 3, 1: 2, 2: 1, 3: 0} } )
def test_get_collection_from_type_from_id(self): ''' Test if the method returns the item with the passed numeric id ''' self.assertEqual( BaseModel.get_collection_from_type( SAMPLE_DATAFRAME.copy(), "from_id", "col_2", 2 ).to_dict(), {"col_1": "b", "col_2": 2, "col_3": 2} )
def test_reform_filters_full(self): ''' Test if filters are classified correctly as pre and post ''' self.assertEqual( BaseModel.reform_filters_for_pandas( ['post-eq-a-1', 'and', 'in-b-2-3', 'and', 'post-nn-c', 'or', 'eq-d-4'] ), ( ['in-b-2-3', 'and', 'eq-d-4'], [['post', 'eq', 'a', '1'], 'and', ['post', 'nn', 'c']] ) )
def test_filter_dataset_nn(self): ''' Test if dataset is filtered by NN clause ''' self.assertEqual( BaseModel.filter_pandas_dataset( SAMPLE_DATAFRAME_NA, [['post', 'nn', 'col_2']] ).to_dict(), { 'col_1': {0: 'd', 1: 'b', 2: 'a'}, 'col_2': {0: 3, 1: 2, 2: 1}, 'col_3': {0: 3, 1: 2, 2: 1} } )
def test_filter_dataset_mixed(self): ''' Test if dataset is filtered by multiple clauses ''' self.assertEqual( BaseModel.filter_pandas_dataset( SAMPLE_DATAFRAME_NA, [['post', 'in', 'col_3', '3', '1'], ['post', 'in', 'col_2', '3', '2']] ).to_dict(), { 'col_1': {0: 'd'}, 'col_2': {0: 3}, 'col_3': {0: 3} } )
def test_filter_dataset_in(self): ''' Test if dataset is filtered by IN clause ''' self.assertEqual( BaseModel.filter_pandas_dataset( SAMPLE_DATAFRAME, [['post', 'in', 'col_2', '3', '2']] ).to_dict(), { 'col_1': {0: 'd', 1: 'b'}, 'col_2': {0: 3, 1: 2}, 'col_3': {0: 3, 1: 2} } )
def test_filter_dataset_eq(self): ''' Test if dataset is filtered by EQ clause ''' self.assertEqual( BaseModel.filter_pandas_dataset( SAMPLE_DATAFRAME, [['post', 'eq', 'col_2', '3']] ).to_dict(), { 'col_1': {0: 'd'}, 'col_2': {0: 3}, 'col_3': {0: 3} } )
def test_build_derivatives(self): ''' Test if derivate object is added to the collection ''' options = {'cd_analysis_unit': 2} rules = { "instances": [{"name": "inst_1", 'type': 'max', 'named_prop': 'col_2'}] } sources = {"dataset": SAMPLE_DATAFRAME.copy()} (der_data, der_anynodata) = BaseModel.build_derivatives( rules, options, sources, {} ) self.assertEqual( der_data["inst_1"].to_dict(), {"col_1": "d", "col_2": 3, "col_3": 3} ) self.assertEqual(der_anynodata, False)
def test_build_derivatives_nodata(self): ''' Test the derivate objects is added with no_data flag ''' options = {'cd_analysis_unit': 99} rules = { "instances": [{"name": "inst_1", 'type': 'from_id', 'named_prop': 'col_2'}] } sources = {"dataset": SAMPLE_DATAFRAME.copy()} (der_data, der_anynodata) = BaseModel.build_derivatives( rules, options, sources, {} ) self.assertEqual( der_data["inst_1"], None ) self.assertEqual(der_anynodata, True)
model_save_dir = opt['save_dir'] + '/' + model_id opt['model_save_dir'] = model_save_dir helper.ensure_dir(model_save_dir, verbose=True) # save config helper.save_config(opt, model_save_dir + '/config.json', verbose=True) vocab.save(model_save_dir + '/vocab.pkl') file_logger = helper.FileLogger( model_save_dir + '/' + opt['log'], header="# epoch\ttrain_loss\tdev_loss\tdev_f1\ttest_loss\ttest_f1") # print model info helper.print_config(opt) # model base_model = BaseModel(opt=opt, emb_matrix=emb_matrix) id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()]) dev_f1_history = [] current_lr = opt['lr'] global_step = 0 global_start_time = time.time() format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}' max_steps = len(train_batch) * opt['num_epoch'] # start training max_dev_f1, max_dev_id = 0, 0 max_test_f1, max_test_id = 0, 0 for epoch in range(1, opt['num_epoch'] + 1): train_loss = 0
def test_default_on_unmapped(self): ''' Verifica se retorna np.mean se agregação não prevista ''' self.assertEqual(BaseModel.aggr_to_np('xpto'), np.mean)
vocab.save(model_save_dir + '/vocab.pkl') file_logger = helper.FileLogger( model_save_dir + '/' + opt['log'], header="# epoch\ttrain_loss\tdev_loss\tdev_f1\ttest_loss\ttest_f1") # print model info helper.print_config(opt) # model base_outputs = ['placeholder' for _ in range(0, len(train_batch))] if opt['base_mode'] == 3: base_outputs = [] base_model_file = opt['save_dir'] + '/' + opt['base_id'] + '/best_model.pt' print("Loading base model from {}".format(base_model_file)) base_opt = torch_utils.load_config(base_model_file) base_model = BaseModel(opt=base_opt) base_model.load(base_model_file) base_model.model.eval() for _, batch in enumerate(train_batch): inputs = [b.cuda() for b in batch[:10] ] if opt['cuda'] else [b for b in batch[:10]] base_logits, _, _ = base_model.model(inputs) base_outputs.append([base_logits.data.cpu().numpy()]) teacher_outputs = [] if opt['use_teacher']: teacher_model_file = opt['save_dir'] + '/' + opt[ 'teacher_id'] + '/best_model.pt' print("Loading teacher model from {}".format(teacher_model_file)) teacher_opt = torch_utils.load_config(teacher_model_file) teacher_model = TeacherModel(opt=teacher_opt)
def test_reform_filters_no_filter(self): ''' Test if None is returned if no filter is sent to the method ''' self.assertEqual( BaseModel.reform_filters_for_pandas(None), (None, None) )
def test_sum(self): ''' Verifica se retorna np.sum ''' self.assertEqual(BaseModel.aggr_to_np('sum'), np.sum)
def test_default_on_none(self): ''' Verifica se retorna np.mean se agregação nula ''' self.assertEqual(BaseModel.aggr_to_np(None), np.mean)
def test_sum_on_count(self): ''' Verifica se não retorna nada e a agregação for count ''' self.assertEqual(BaseModel.aggr_to_np('count'), np.sum)
def test_default_on_empty(self): ''' Verifica se retorna np.mean se agragação vazia ''' self.assertEqual(BaseModel.aggr_to_np(''), np.mean)
def test_single_aggr(self): ''' Verifica se retorna np.mean se agregação nula ''' model = BaseModel() aggrs = 'sum' self.assertEqual(model.convert_aggr_to_np(aggrs, None), np.sum)