def test_correct_total_sample_size_and_counts_and_mutability(self):
data = [['test1', 1.0], ['test2', 2.0], ['test3', 3.0], [None, None],
['test5', 5.0], ['test6', 6.0], [None, None], ['test7', 7.0]]
data = pd.DataFrame(data, columns=['NAME', 'VALUE'])
profiler_options = ProfilerOptions()
profiler_options.set({'data_labeler.is_enabled': False})
col_one_len = len(data['NAME'])
col_two_len = len(data['VALUE'])
# Test reloading data, ensuring immutable
for i in range(2):
# Profile Once
data.index = pd.RangeIndex(0, 8)
profile = dp.Profiler(data,
profiler_options=profiler_options,
samples_per_update=2)
# Profile Twice
data.index = pd.RangeIndex(8, 16)
profile.update_profile(data)
# rows sampled are [5, 6], [13, 14] (0 index)
self.assertEqual(16, profile.total_samples)
self.assertEqual(4, profile._max_col_samples_used)
self.assertEqual(2, profile.row_has_null_count)
self.assertEqual(0.5, profile._get_row_has_null_ratio())
self.assertEqual(2, profile.row_is_null_count)
self.assertEqual(0.5, profile._get_row_is_null_ratio())
self.assertEqual(0.4375, profile._get_unique_row_ratio())
self.assertEqual(9, profile._get_duplicate_row_count())
self.assertEqual(col_one_len, len(data['NAME']))
self.assertEqual(col_two_len, len(data['VALUE']))
def test_correct_rows_ingested(self):
test_dict = {
'1': ['nan', 'null', None, None, ''],
1: ['nan', 'None', 'null', None, ''],
}
test_dataset = pd.DataFrame(data=test_dict)
profiler_options = ProfilerOptions()
profiler_options.set({'data_labeler.is_enabled': False})
trained_schema = dp.Profiler(test_dataset,
len(test_dataset),
profiler_options=profiler_options)
self.assertCountEqual(['', 'nan', 'None', 'null'],
trained_schema.profile['1'].null_types)
self.assertEqual(5, trained_schema.profile['1'].null_count)
self.assertEqual({
'': {4},
'nan': {0},
'None': {2, 3},
'null': {1}
}, trained_schema.profile['1'].null_types_index)
self.assertCountEqual(['', 'nan', 'None', 'null'],
trained_schema.profile[1].null_types)
self.assertEqual(5, trained_schema.profile[1].null_count)
self.assertEqual({
'': {4},
'nan': {0},
'None': {1, 3},
'null': {2}
}, trained_schema.profile[1].null_types_index)
def test_sample_size_passed_to_profile(self, *mocks):
update_mock = mocks[0]
# data setup
data = pd.DataFrame([0] * int(50e3))
# option setup
profiler_options = ProfilerOptions()
profiler_options.structured_options.multiprocess.is_enabled = False
profiler_options.set({'data_labeler.is_enabled': False})
# test data size < min_sample_size = 5000 by default
profiler = dp.Profiler(data[:1000], profiler_options=profiler_options)
profiler._min_sample_size = 5000
profiler._sampling_ratio = 0.2
self.assertEqual(1000, update_mock.call_args[0][1])
# test data size * 0.20 < min_sample_size < data size
profiler = dp.Profiler(data[:10000], profiler_options=profiler_options)
profiler._min_sample_size = 5000
profiler._sampling_ratio = 0.2
self.assertEqual(5000, update_mock.call_args[0][1])
# test min_sample_size > data size * 0.20
profiler = dp.Profiler(data, profiler_options=profiler_options)
profiler._min_sample_size = 5000
profiler._sampling_ratio = 0.2
self.assertEqual(10000, update_mock.call_args[0][1])
def setUpClass(cls):
cls.input_file_path = os.path.join(test_root_path, 'data',
'csv/aws_honeypot_marx_geo.csv')
cls.aws_dataset = pd.read_csv(cls.input_file_path)
profiler_options = ProfilerOptions()
profiler_options.set({'data_labeler.is_enabled': False})
cls.trained_schema = dp.Profiler(cls.aws_dataset,
len(cls.aws_dataset),
profiler_options=profiler_options)
def test_correct_null_row_counts(self):
file_path = os.path.join(test_root_path, 'data', 'csv/empty_rows.txt')
data = pd.read_csv(file_path)
profiler_options = ProfilerOptions()
profiler_options.set({'data_labeler.is_enabled': False})
profile = dp.Profiler(data, profiler_options=profiler_options)
self.assertEqual(2, profile.row_has_null_count)
self.assertEqual(0.25, profile._get_row_has_null_ratio())
self.assertEqual(2, profile.row_is_null_count)
self.assertEqual(0.25, profile._get_row_is_null_ratio())
file_path = os.path.join(test_root_path, 'data','csv/iris-with-null-rows.csv')
data = pd.read_csv(file_path)
profile = dp.Profiler(data, profiler_options=profiler_options)
self.assertEqual(13, profile.row_has_null_count)
self.assertEqual(13/24, profile._get_row_has_null_ratio())
self.assertEqual(3, profile.row_is_null_count)
self.assertEqual(3/24, profile._get_row_is_null_ratio())
def test_null_in_file(self):
filename_null_in_file = os.path.join(
test_root_path, 'data', 'csv/sparse-first-and-last-column.txt')
profiler_options = ProfilerOptions()
profiler_options.set({'data_labeler.is_enabled': False})
data = dp.Data(filename_null_in_file)
profile = dp.Profiler(data, profiler_options=profiler_options)
report = profile.report(report_options={"output_format":"pretty"})
self.assertEqual(
report['data_stats']['COUNT']['statistics']['null_types_index'],
{'': '[2, 3, 4, 5, 7, 8]'}
)
self.assertEqual(
report['data_stats'][' NUMBERS']['statistics']['null_types_index'],
{'': '[5, 6, 8]', ' ': '[2, 4]'}
)
def get_options(self, **params):
options = ProfilerOptions()
options.set(params)
return options