Ejemplo n.º 1
0
    def test_wrapping(self):
        ''' Verifica se retorna o dataset apenas com o wrapping '''
        model = BaseModel()

        options = {
            **{
                "categorias": [
                    'nm_indicador', 'nu_competencia', 'vl_indicador', 'lat_mun', 'long_mun'
                ],
                "pivot": None
            }, **COMMON_OPTIONS
        }

        result = "".join(model.wrap_result(SAMPLE_DATAFRAME.copy(), options).split())

        str_expected = COMMON_EXPECTED_RESPONSE_STRING.format(
            """
                "nm_indicador": "Ficticio",
                "nu_competencia": 2099,
                "vl_indicador": 1.0
            },
            {
                "nm_indicador": "Ficticio",
                "nu_competencia": 2047,
                "vl_indicador": 0.5
            """
        )
        expected = "".join(str_expected.split())

        self.assertEqual(result, expected)
Ejemplo n.º 2
0
    def test_set_cache(self):
        """Test BaseModel.set_cache"""

        self.assertIsInstance(BaseModel._cache, FileSystemCache)
        old_cache = BaseModel._cache
        BaseModel.set_cache(NullCache())
        self.assertIsInstance(BaseModel._cache, NullCache)
        SampleModel.set_cache(old_cache)
        self.assertIsInstance(BaseModel._cache, NullCache)
        self.assertIsInstance(SampleModel._cache, FileSystemCache)
Ejemplo n.º 3
0
    def test_set_cache(self):
        """Test BaseModel.set_cache"""

        self.assertIsInstance(BaseModel._cache, FileSystemCache)
        old_cache = BaseModel._cache
        BaseModel.set_cache(NullCache())
        self.assertIsInstance(BaseModel._cache, NullCache)
        SampleModel.set_cache(old_cache)
        self.assertIsInstance(BaseModel._cache, NullCache)
        self.assertIsInstance(SampleModel._cache, FileSystemCache)
Ejemplo n.º 4
0
    def setUp(self):
        """BaseModel test set up"""

        if os.path.isfile("/tmp/box.db"):
            os.unlink("/tmp/box.db")
        DBHelper().set_db("/tmp/box.db")
        InstallHelper.reset()
        cache = FileSystemCache("/tmp/werkzeug")
        cache.clear()
        BaseModel.set_cache(cache)
        SampleModel.install()
Ejemplo n.º 5
0
    def setUp(self):
        """BaseModel test set up"""

        if os.path.isfile('/tmp/box.db'):
            os.unlink('/tmp/box.db')
        DBHelper().set_db('/tmp/box.db')
        InstallHelper.reset()
        cache = FileSystemCache('/tmp/werkzeug')
        cache.clear()
        BaseModel.set_cache(cache)
        SampleModel.install()
Ejemplo n.º 6
0
 def test_multiple_aggr(self):
     ''' Verifica se retorna np.mean se agregação nula '''
     model = BaseModel()
     aggrs = ['sum', 'count', 'xpto', '']
     vals = ['vl_indicador', 'cd_municipio', 'nu_competencia',
             'another_field']
     expected = {
         'vl_indicador': np.sum,
         'cd_municipio': np.sum,
         'nu_competencia': np.mean,
         'another_field': np.mean
     }
     self.assertEqual(model.convert_aggr_to_np(aggrs, vals), expected)
 def test_no_def(self):
     ''' Verifica se retorna np.mean se agregação nula '''
     model = BaseModel()
     self.assertRaises(
         NotImplementedError,
         model.get_repo
     )
Ejemplo n.º 8
0
class Document(BaseModel):
    __tablename__ = 'documents'

    content = db.Column('content', db.Text(), nullable=False)
    title = db.Column('title', db.String(200), nullable=False)
    description = db.Column('description', db.Text(), nullable=True)
    author = db.Column('author', db.String(200), nullable=True)

    # Table metadata can be specified as follows -
    __table_args__ = (
        db.UniqueConstraint('title', 'is_deleted'),
        db.Index(BaseModel.create_index(__tablename__, 'title', 'is_deleted'),
                 'title', 'is_deleted'),
    )

    def __str__(self):
        return self.title

    def __repr__(self):
        return json.dumps(self.to_dict())

    def to_dict(self, *args, **kwargs):
        return {
            'id': self.id,
            'author': self.author,
            'title': self.title,
            'description': self.description,
            'content': self.content
        }
Ejemplo n.º 9
0
 def test_get_collection_from_invalid_type(self):
     ''' Test if the method returns None if an invalid type is passed '''
     self.assertEqual(
         BaseModel.get_collection_from_type(
             SAMPLE_DATAFRAME.copy(),
             'invalid'
         ),
         None
     )
Ejemplo n.º 10
0
 def test_resort_dataset_desc(self):
     ''' Test if dataset is ordered descending if all args are descending '''
     self.assertEqual(
         BaseModel.resort_dataset(
             SAMPLE_DATAFRAME,
             ["-col_1", "-col_2", "-col_3"]
         ).to_dict('split')['data'],
         [['d', 3, 3], ['c', 0, 0], ['b', 2, 2], ['a', 1, 1]]
     )
Ejemplo n.º 11
0
 def test_get_collection_from_type_first_occurence(self):
     ''' Test if the method returns the first item '''
     self.assertEqual(
         BaseModel.get_collection_from_type(
             SAMPLE_DATAFRAME.copy(),
             "first_occurence"
         ).to_dict(),
         {"index": 0, "col_1": "d", "col_2": 3, "col_3": 3}
     )
Ejemplo n.º 12
0
 def test_resort_dataset_no_rule(self):
     ''' Test if dataset resturns as is if no order rule is sent '''
     self.assertEqual(
         BaseModel.resort_dataset(
             SAMPLE_DATAFRAME,
             ["col_1", "col_2"]
         ).to_dict('split')['data'],
         [['a', 1, 1], ['b', 2, 2], ['c', 0, 0], ['d', 3, 3]]
     )
Ejemplo n.º 13
0
 def test_get_collection_from_missing_type(self):
     ''' Test if the method returns None if no type is passed '''
     self.assertEqual(
         BaseModel.get_collection_from_type(
             SAMPLE_DATAFRAME.copy(),
             None
         ),
         None
     )
Ejemplo n.º 14
0
 def test_get_collection_from_type_max(self):
     ''' Test if the method returns the item with maximum value in colum '''
     self.assertEqual(
         BaseModel.get_collection_from_type(
             SAMPLE_DATAFRAME.copy(),
             "max",
             "col_2"
         ).to_dict(),
         {"col_1": "d", "col_2": 3, "col_3": 3}
     )
Ejemplo n.º 15
0
 def test_filter_dataset_nofilter(self):
     ''' Test if dataset returns when no filter is actually sent '''
     self.assertEqual(
         BaseModel.filter_pandas_dataset(SAMPLE_DATAFRAME, None).to_dict(),
         {
             'col_1': {0: 'd', 1: 'b', 2: 'a', 3: 'c'},
             'col_2': {0: 3, 1: 2, 2: 1, 3: 0},
             'col_3': {0: 3, 1: 2, 2: 1, 3: 0}
         }
     )
Ejemplo n.º 16
0
 def test_get_collection_from_type_from_id(self):
     ''' Test if the method returns the item with the passed numeric id '''
     self.assertEqual(
         BaseModel.get_collection_from_type(
             SAMPLE_DATAFRAME.copy(),
             "from_id",
             "col_2",
             2
         ).to_dict(),
         {"col_1": "b", "col_2": 2, "col_3": 2}
     )
Ejemplo n.º 17
0
 def test_reform_filters_full(self):
     ''' Test if filters are classified correctly as pre and post '''
     self.assertEqual(
         BaseModel.reform_filters_for_pandas(
             ['post-eq-a-1', 'and', 'in-b-2-3', 'and', 'post-nn-c', 'or', 'eq-d-4']
         ),
         (
             ['in-b-2-3', 'and', 'eq-d-4'],
             [['post', 'eq', 'a', '1'], 'and', ['post', 'nn', 'c']]
         )
     )
Ejemplo n.º 18
0
 def test_filter_dataset_nn(self):
     ''' Test if dataset is filtered by NN clause '''
     self.assertEqual(
         BaseModel.filter_pandas_dataset(
             SAMPLE_DATAFRAME_NA,
             [['post', 'nn', 'col_2']]
         ).to_dict(),
         {
             'col_1': {0: 'd', 1: 'b', 2: 'a'},
             'col_2': {0: 3, 1: 2, 2: 1},
             'col_3': {0: 3, 1: 2, 2: 1}
         }
     )
Ejemplo n.º 19
0
 def test_filter_dataset_mixed(self):
     ''' Test if dataset is filtered by multiple clauses '''
     self.assertEqual(
         BaseModel.filter_pandas_dataset(
             SAMPLE_DATAFRAME_NA,
             [['post', 'in', 'col_3', '3', '1'], ['post', 'in', 'col_2', '3', '2']]
         ).to_dict(),
         {
             'col_1': {0: 'd'},
             'col_2': {0: 3},
             'col_3': {0: 3}
         }
     )
Ejemplo n.º 20
0
 def test_filter_dataset_in(self):
     ''' Test if dataset is filtered by IN clause '''
     self.assertEqual(
         BaseModel.filter_pandas_dataset(
             SAMPLE_DATAFRAME,
             [['post', 'in', 'col_2', '3', '2']]
         ).to_dict(),
         {
             'col_1': {0: 'd', 1: 'b'},
             'col_2': {0: 3, 1: 2},
             'col_3': {0: 3, 1: 2}
         }
     )
Ejemplo n.º 21
0
 def test_filter_dataset_eq(self):
     ''' Test if dataset is filtered by EQ clause '''
     self.assertEqual(
         BaseModel.filter_pandas_dataset(
             SAMPLE_DATAFRAME,
             [['post', 'eq', 'col_2', '3']]
         ).to_dict(),
         {
             'col_1': {0: 'd'},
             'col_2': {0: 3},
             'col_3': {0: 3}
         }
     )
Ejemplo n.º 22
0
 def test_build_derivatives(self):
     ''' Test if derivate object is added to the collection '''
     options = {'cd_analysis_unit': 2}
     rules = {
         "instances": [{"name": "inst_1", 'type': 'max', 'named_prop': 'col_2'}]
     }
     sources = {"dataset": SAMPLE_DATAFRAME.copy()}
     (der_data, der_anynodata) = BaseModel.build_derivatives(
         rules,
         options,
         sources,
         {}
     )
     self.assertEqual(
         der_data["inst_1"].to_dict(),
         {"col_1": "d", "col_2": 3, "col_3": 3}
     )
     self.assertEqual(der_anynodata, False)
Ejemplo n.º 23
0
 def test_build_derivatives_nodata(self):
     ''' Test the derivate objects is added with no_data flag '''
     options = {'cd_analysis_unit': 99}
     rules = {
         "instances": [{"name": "inst_1", 'type': 'from_id', 'named_prop': 'col_2'}]
     }
     sources = {"dataset": SAMPLE_DATAFRAME.copy()}
     (der_data, der_anynodata) = BaseModel.build_derivatives(
         rules,
         options,
         sources,
         {}
     )
     self.assertEqual(
         der_data["inst_1"],
         None
     )
     self.assertEqual(der_anynodata, True)
Ejemplo n.º 24
0
model_save_dir = opt['save_dir'] + '/' + model_id
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)

# save config
helper.save_config(opt, model_save_dir + '/config.json', verbose=True)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
    model_save_dir + '/' + opt['log'],
    header="# epoch\ttrain_loss\tdev_loss\tdev_f1\ttest_loss\ttest_f1")

# print model info
helper.print_config(opt)

# model
base_model = BaseModel(opt=opt, emb_matrix=emb_matrix)

id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
dev_f1_history = []
current_lr = opt['lr']

global_step = 0
global_start_time = time.time()
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
max_steps = len(train_batch) * opt['num_epoch']

# start training
max_dev_f1, max_dev_id = 0, 0
max_test_f1, max_test_id = 0, 0
for epoch in range(1, opt['num_epoch'] + 1):
    train_loss = 0
Ejemplo n.º 25
0
 def test_default_on_unmapped(self):
     ''' Verifica se retorna np.mean se agregação não prevista '''
     self.assertEqual(BaseModel.aggr_to_np('xpto'), np.mean)
Ejemplo n.º 26
0
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
    model_save_dir + '/' + opt['log'],
    header="# epoch\ttrain_loss\tdev_loss\tdev_f1\ttest_loss\ttest_f1")

# print model info
helper.print_config(opt)

# model
base_outputs = ['placeholder' for _ in range(0, len(train_batch))]
if opt['base_mode'] == 3:
    base_outputs = []
    base_model_file = opt['save_dir'] + '/' + opt['base_id'] + '/best_model.pt'
    print("Loading base model from {}".format(base_model_file))
    base_opt = torch_utils.load_config(base_model_file)
    base_model = BaseModel(opt=base_opt)
    base_model.load(base_model_file)
    base_model.model.eval()
    for _, batch in enumerate(train_batch):
        inputs = [b.cuda() for b in batch[:10]
                  ] if opt['cuda'] else [b for b in batch[:10]]
        base_logits, _, _ = base_model.model(inputs)
        base_outputs.append([base_logits.data.cpu().numpy()])

teacher_outputs = []
if opt['use_teacher']:
    teacher_model_file = opt['save_dir'] + '/' + opt[
        'teacher_id'] + '/best_model.pt'
    print("Loading teacher model from {}".format(teacher_model_file))
    teacher_opt = torch_utils.load_config(teacher_model_file)
    teacher_model = TeacherModel(opt=teacher_opt)
Ejemplo n.º 27
0
 def test_reform_filters_no_filter(self):
     ''' Test if None is returned if no filter is sent to the method '''
     self.assertEqual(
         BaseModel.reform_filters_for_pandas(None),
         (None, None)
     )
Ejemplo n.º 28
0
 def test_sum(self):
     ''' Verifica se retorna np.sum '''
     self.assertEqual(BaseModel.aggr_to_np('sum'), np.sum)
Ejemplo n.º 29
0
 def test_default_on_none(self):
     ''' Verifica se retorna np.mean se agregação nula '''
     self.assertEqual(BaseModel.aggr_to_np(None), np.mean)
Ejemplo n.º 30
0
 def test_sum_on_count(self):
     ''' Verifica se não retorna nada e a agregação for count '''
     self.assertEqual(BaseModel.aggr_to_np('count'), np.sum)
Ejemplo n.º 31
0
 def test_default_on_empty(self):
     ''' Verifica se retorna np.mean se agragação vazia '''
     self.assertEqual(BaseModel.aggr_to_np(''), np.mean)
Ejemplo n.º 32
0
 def test_single_aggr(self):
     ''' Verifica se retorna np.mean se agregação nula '''
     model = BaseModel()
     aggrs = 'sum'
     self.assertEqual(model.convert_aggr_to_np(aggrs, None), np.sum)