def test_default_on_none(self):
''' Verifica se retorna np.mean se agregação nula '''
self.assertEqual(
StubEmpresa().get_stats_local_options(
{
'column': 2099,
'cnpj_raiz': '12345678',
'cnpj': '12345678000101',
'id_pf': '12345678900'
},
{
'compet': 'col_compet',
'cnpj_raiz': 'col_cnpj_raiz',
'cnpj_raiz_flag': 'flag', 'cnpj': 'cnpj_col',
'cnpj_flag': 'flag_2', 'pf': 'pf_col'
},
'theme',
None
),
{
'categorias': ['col_cnpj_raiz'],
'agregacao': ['count'],
'where': [
"eq-col_cnpj_raiz-12345678",
"and", "eq-flag-'1'",
"and", "eq-cnpj_col-12345678000101",
"and", "eq-flag_2-'1'",
"and", "eq-pf_col-12345678900"
],
'theme': 'theme'
}
)
def test_get_grouped_stats(self):
''' Tests basic grouped stats retrieval '''
self.assertEqual(
StubEmpresa().get_grouped_stats(
{},
{'theme': 'mytheme'},
{'compet': 'compet'}
),
{
'stats_estab': {
'12345678000101': {'agr_count': 100, 'compet': 2047},
'12345678000202': {'agr_count': 200, 'compet': 2099}
},
'stats_compet': {
'2047': {'agr_count': 100, 'cnpj': '12345678000101'},
'2099': {'agr_count': 200, 'cnpj': '12345678000202'}
},
'stats_estab_compet': {
'2047_12345678000101': {
'agr_count': 100, 'cnpj': '12345678000101', 'compet': 2047
},
'2099_12345678000202': {
'agr_count': 200, 'cnpj': '12345678000202', 'compet': 2099
}
}
}
)
def test_get_statistic_column_family_no_perspective(self):
''' Tests statistics retrieval for a specific dataset with no
perspective definition, but an existing timeframe rule '''
self.assertEqual(
StubEmpresa().get_statistics({
"cnpj_raiz": '12345678',
"theme": 'rais',
"column_family": "rais",
"categorias": ['nu_cnpj_cei']
}),
{
"rais": {
'fonte': 'Fonte',
'stats': {'nu_cnpj_cei': '12345678000101', 'nu_ano_rais': 2047, 'agr_count': 100},
'stats_estab': {
'12345678000101': {'agr_count': 100, 'nu_ano_rais': 2047},
'12345678000202': {'agr_count': 200, 'nu_ano_rais': 2099}
},
'stats_compet': {
'2047': {'agr_count': 100, 'nu_cnpj_cei': '12345678000101'},
'2099': {'agr_count': 200, 'nu_cnpj_cei': '12345678000202'}
},
'stats_estab_compet': {
'2047_12345678000101': {
'agr_count': 100, 'nu_cnpj_cei': '12345678000101', 'nu_ano_rais': 2047
},
'2099_12345678000202': {
'agr_count': 200, 'nu_cnpj_cei': '12345678000202', 'nu_ano_rais': 2099
}
}
}
}
)
def test_is_valid_loading_entry_wrong_column_family(self):
''' Tests if error is raised when an incorrect column family is given '''
self.assertRaises(
ValueError,
StubEmpresa().is_valid_loading_entry,
'12345678',
{'column_family': 'non-existent'}
)
def test_is_valid_loading_entry_no_column_family(self):
''' Tests if error is raised when no column family is given '''
self.assertRaises(
ValueError,
StubEmpresa().is_valid_loading_entry,
'12345678',
{}
)
def test_is_valid_loading_entry_no_options(self):
''' Tests if error is raised when no options is given '''
self.assertRaises(
ValueError,
StubEmpresa().is_valid_loading_entry,
'12345678',
None
)
def test_is_valid_loading_entry_wrong_column(self):
''' Tests if error is raised when an incorrect column is given
to filter the data from a row '''
self.assertRaises(
ValueError,
StubEmpresa().is_valid_loading_entry,
'12345678',
{'column_family': 'test', 'column': '2099'}
)
def test_is_valid_loading_entry_false_expired(self):
''' Tests if validation fails '''
self.assertEqual(
StubEmpresa().is_valid_loading_entry(
'12345678',
{'column_family': 'expired', 'column': '2018'}
),
False
)
def test_get_loading_entry_empty_options(self):
''' Tests if no column status is returned when an empty options is given '''
self.assertEqual(
StubEmpresa().get_loading_entry('12345678', {}),
(
self.get_expected_status_dict(),
None
)
)
def test_is_valid_loading_entry_false_by_status(self):
''' Tests if validation fails '''
self.assertEqual(
StubEmpresa().is_valid_loading_entry(
'12345678',
{'column_family': 'failed_status', 'column': '2017'}
),
False
)
def test_is_valid_loading_entry(self):
''' Tests if validation passes whe no falseability conditions are
met '''
self.assertEqual(
StubEmpresa().is_valid_loading_entry(
'12345678',
{'column_family': 'test'}
),
True
)
def test_get_statistic_overall_timeframe_validation(self):
''' Tests if timeframe requirement is forced on overall request '''
self.assertRaises(
AttributeError,
StubEmpresa().get_statistics,
{
"categorias": ['cnpj'],
'where': []
}
)
def test_get_grouped_stats_displaced_compet_alt(self):
''' Tests basic grouped stats retrieval '''
self.assertEqual(
StubEmpresa().get_grouped_stats(
{},
{'theme': 'catewb'},
{}
),
StubEmpresa.EXPECTED_GROUPED_STATS
)
def test_is_valid_loading_entry_false_column(self):
''' Tests if validation fails when the requires column is not in the
available list '''
self.assertEqual(
StubEmpresa().is_valid_loading_entry(
'12345678',
{'column_family': 'another'}
),
False
)
def test_assess_column_status_no_column_args(self):
''' Tests if UNAVAILABLE is returned when no column args are provided '''
self.assertEqual(
StubEmpresa.assess_column_status(
['2017', '2018'],
None,
None
),
'UNAVAILABLE'
)
def test_assess_column_status_from_data(self):
''' Tests if a regular status is passed AS-IS if guard conditions
are not met '''
self.assertEqual(
StubEmpresa.assess_column_status(
['2017', '2018'],
{"2018": 'FAILED|2000-01-01'},
'2018'
),
'FAILED|2000-01-01'
)
def test_assess_column_status_deprecated(self):
''' Tests if MISSING is returned when a given column is not expected
(removed from the datalake reference, but is retrieved from REDIS '''
self.assertEqual(
StubEmpresa.assess_column_status(
['2017', '2018'],
{"2019": 'INGESTED'},
'2019'
),
'DEPRECATED'
)
def test_assess_column_status_missing(self):
''' Tests if MISSING is returned when a given column is expected but
not present in the columns_available (from REDIS) '''
self.assertEqual(
StubEmpresa.assess_column_status(
['2017', '2018'],
{"2017": 'INGESTED'},
'2018'
),
'MISSING'
)
def test_assess_column_status_empty_available_column_arg(self):
''' Tests if UNAVAILABLE is returned when no column and an empty
columns_available args are provided '''
self.assertEqual(
StubEmpresa.assess_column_status(
['2017', '2018'],
{},
None
),
'UNAVAILABLE'
)
def test_assess_column_status_unavailable_deprecated_not_ingested(self):
''' Tests if MISSING is returned when a given column is not expected
(removed from the datalake reference, but is retrieved from REDIS
with a non-INGESTED status '''
self.assertEqual(
StubEmpresa.assess_column_status(
['2017', '2018'],
{"2019": 'FAILED|2000-01-01'},
'2019'
),
'UNAVAILABLE'
)
def test_valid_loading_entry_no_column_in_options(self):
''' Tests return of valid loading entry when no column is given '''
self.assertEqual(
StubEmpresa().get_loading_entry(
'12345678',
{'column_family': 'failed_status'}
),
(
self.get_expected_status_dict(),
None
)
)
def test_valid_loading_entry(self):
''' Tests return of valid ingested loading entry '''
self.assertEqual(
StubEmpresa().get_loading_entry(
'12345678',
{'column_family': 'test', 'column': '2017'}
),
(
self.get_expected_status_dict(),
f'INGESTED|{datetime.strftime(datetime.now(), "%Y-%m-%d")}'
)
)
def test_get_statistic_raise_on_timeframe_validation_error(self):
''' Tests if timeframe requirement is forced '''
self.assertRaises(
AttributeError,
StubEmpresa().get_statistics,
{
"theme": "auto",
"column_family": "auto",
"categorias": ['cnpj'],
'where': []
}
)
def test_get_statistic_from_perspective_perspective(self):
''' Tests basic grouped perspective stats retrieval '''
self.assertEqual(
StubEmpresa().get_statistics_from_perspective(
'mytheme', 'mypersp', {},
{"categorias": ['cnpj'], 'where': []},
{"categorias": ['cnpj']}
),
{
**{
'stats': {'cnpj': '12345678000101', 'compet': 2047, 'agr_count': 100}
},
**StubEmpresa.EXPECTED_GROUPED_STATS
}
)
def test_get_statistic_column_family_with_perspective_invalid_value(self):
''' Tests statistics retrieval for a specific dataset with
perspective and timeframe definitions '''
self.assertRaises(
AttributeError,
StubEmpresa().get_statistics,
{
"theme": 'catweb',
"column_family": "catweb",
"perspective": 'invalid',
"categorias": ['cnpj'],
"column": '2047',
'where': []
}
)
def test_get_statistic_column_family_no_perspective_no_timeframe(self):
''' Tests statistics retrieval for a specific dataset with no
perspective and timeframe definition '''
self.assertEqual(
StubEmpresa().get_statistics({
"cnpj_raiz": '12345678',
"theme": 'rfb',
"column_family": "rfb",
"categorias": ['cnpj']
}),
{
"rfb": {
'fonte': 'Fonte',
'stats': {'cnpj': '12345678000101', 'compet': 2047, 'agr_count': 100}
}
}
)