def test_math_single_string_occurring_once(self): list_1 = [('2', 2)] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a list_1 = [('2', 2)] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a
def test_math_single_string_occurring_once(self): list_1 = [('2', 2)] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a list_1 = [('2', 2)] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a
def test_math_identical_single_float_occuring_once(self): list_1 = [(2.0, 1)] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a list_2 = [(2.5, 1)] list_2a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_2, 2.5) == list_2a assert mod.get_variance_and_stddev(list_2) == list_2a
def test_math_identical_single_float_occuring_once(self): list_1 = [(2.0, 1)] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a list_2 = [(2.5, 1)] list_2a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_2, 2.5) == list_2a assert mod.get_variance_and_stddev(list_2) == list_2a
def test_math_floats_multiple_floats_occuring_once(self): list_1 = [(2.0, 1), (3.0, 1), (4.0, 1)] list_1a = ('0.67', '0.82') var, stddev = mod.get_variance_and_stddev(list_1) small_var = self._convert_float(var) small_stddev = self._convert_float(stddev) assert (small_var, small_stddev) == list_1a
def test_math_multiple_numbers_occuring_once(self): list_1 = [(2, 1), (3, 1), (4, 1)] list_1a = ('0.67', '0.82') var, stddev = mod.get_variance_and_stddev(list_1) small_var = self._convert_float(var) small_stddev = self._convert_float(stddev) assert (small_var, small_stddev) == list_1a list_2 = [(2, 1), (3, 1), (9, 1), (12, 1), (13, 1), (15, 1), (17, 1), (19, 1), (22, 1), (23, 1), (25, 1)] list_2a = ('53.88', '7.34') var, stddev = mod.get_variance_and_stddev(list_2) small_var = self._convert_float(var) small_stddev = self._convert_float(stddev) assert (small_var, small_stddev) == list_2a
def test_math_multiple_numbers_occuring_once(self): list_1 = [(2, 1), (3, 1), (4, 1)] list_1a = ('0.67', '0.82') var, stddev = mod.get_variance_and_stddev(list_1) small_var = self._convert_float(var) small_stddev = self._convert_float(stddev) assert (small_var, small_stddev) == list_1a list_2 = [(2, 1), (3, 1), (9, 1), (12, 1), (13, 1), (15, 1), (17, 1), (19, 1), (22, 1), (23, 1), (25, 1)] list_2a = ('53.88', '7.34') var, stddev = mod.get_variance_and_stddev(list_2) small_var = self._convert_float(var) small_stddev = self._convert_float(stddev) assert (small_var, small_stddev) == list_2a
def test_math_floats_multiple_floats_occuring_once(self): list_1 = [(2.0, 1), (3.0, 1), (4.0, 1)] list_1a = ('0.67', '0.82') var, stddev = mod.get_variance_and_stddev(list_1) small_var = self._convert_float(var) small_stddev = self._convert_float(stddev) assert (small_var, small_stddev) == list_1a
def test_math_multiple_numbers_occurring_multiple_times(self): list_4 = [(2, 10), (3, 15), (9, 10), (12, 7), (13, 4), (15, 2), (17, 1), (19, 1), (22, 1), (23, 1), (25, 1)] list_4a = ('37.11', '6.09') var, stddev = mod.get_variance_and_stddev(list_4) small_var = self._convert_float(var) small_stddev = self._convert_float(stddev) assert (small_var, small_stddev) == list_4a
def test_math_multiple_numbers_occurring_multiple_times(self): list_4 = [(2, 10), (3, 15), (9, 10), (12, 7), (13, 4), (15, 2), (17, 1), (19, 1), (22, 1), (23, 1), (25, 1)] list_4a = ('37.11', '6.09') var, stddev = mod.get_variance_and_stddev(list_4) small_var = self._convert_float(var) small_stddev = self._convert_float(stddev) assert (small_var, small_stddev) == list_4a
def analyze_fields(self, field_number: Optional[int] = None, field_types_overrides: Optional[Dict[int, str]] = None, max_freq_number: Optional[int] = None, read_limit: int = -1) -> None: """ Determines types, names, and characteristics of fields. Arguments: - field_number: if None, then analyzes all fields, otherwise analyzes just the single field (based on zero-offset) - field_types_overrides: - max_freq_number: limits size of collected frequency distribution, allowing for faster analysis or analysis of very large high-cardinality fields. - read_limit: a performance setting that stops file reads after this number. The default is -1 which means 'no limit'. Returns: - Nothing directly - populates instance variables. """ self.max_freq_number = max_freq_number if self.verbose: print('Field Analysis Progress: ') for f_no in range(self.field_cnt): if field_number is not None: # optional analysis of a single field if f_no != field_number: continue if self.verbose: print(' Analyzing field: %d' % f_no) self.field_names[f_no] = miscer.get_field_name(self.filename, self.dialect, f_no) if max_freq_number is None: if field_number is None: max_items = MAX_FREQ_MULTI_COL_DEFAULT else: max_items = MAX_FREQ_SINGLE_COL_DEFAULT else: max_items = max_freq_number (self.field_freqs[f_no], self.field_trunc[f_no], self.field_rows_invalid[f_no]) = miscer.get_field_freq(self.filename, self.dialect, f_no, max_items, read_limit) field_freqs = list(self.field_freqs[f_no].items()) self.field_types[f_no] = typer.get_field_type(self.field_freqs[f_no]) if field_types_overrides: for col_no in field_types_overrides: self.field_types[col_no] = field_types_overrides[col_no] self.field_max[f_no] = miscer.get_max(self.field_types[f_no], field_freqs) self.field_min[f_no] = miscer.get_min(self.field_types[f_no], field_freqs) if self.field_types[f_no] == 'string': self.field_case[f_no] = miscer.get_case(self.field_types[f_no], field_freqs) self.field_min_length[f_no] = miscer.get_min_length(field_freqs) self.field_max_length[f_no] = miscer.get_max_length(field_freqs) self.field_mean_length[f_no] = mather.get_mean_length(field_freqs) else: self.field_case[f_no] = None self.field_min_length[f_no] = None self.field_max_length[f_no] = None self.field_mean_length[f_no] = None if self.field_types[f_no] in ('integer', 'float'): self.field_mean[f_no] = mather.get_mean(field_freqs) self.field_median[f_no] = mather.get_median(field_freqs) (self.variance[f_no], self.stddev[f_no]) \ = mather.get_variance_and_stddev(field_freqs, self.field_mean[f_no]) else: self.field_mean[f_no] = None self.field_median[f_no] = None self.variance[f_no] = None self.stddev[f_no] = None
def test_math_empty_list(self): list_1 = [] list_1a = (None, None) assert mod.get_variance_and_stddev(list_1) == list_1a
def test_math_ignoring_bad_value(self): list_1 = [(2, 2), (3, 'foo')] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a
def test_math_ignoring_bad_key(self): list_1 = [(2, 2), ('bar', 2)] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a
def test_math_single_float_occurring_multiple_times(self): list_1 = [(2.5, 2)] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2.5) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a
def test_math_single_float_occurring_multiple_times(self): list_1 = [(2.5, 2)] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2.5) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a
def test_math_ignoring_bad_key(self): list_1 = [(2, 2), ('bar', 2)] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a
def analyze_fields(self, field_number: Optional[int] = None, field_types_overrides: Optional[Dict[int, str]] = None, max_freq_number: Optional[int] = None, read_limit: int = -1) -> None: """ Determines types, names, and characteristics of fields. Arguments: - field_number: if None, then analyzes all fields, otherwise analyzes just the single field (based on zero-offset) - field_types_overrides: - max_freq_number: limits size of collected frequency distribution, allowing for faster analysis or analysis of very large high-cardinality fields. - read_limit: a performance setting that stops file reads after this number. The default is -1 which means 'no limit'. Returns: - Nothing directly - populates instance variables. """ self.max_freq_number = max_freq_number if self.verbose: print('Field Analysis Progress: ') for f_no in range(self.field_cnt): if field_number is not None: # optional analysis of a single field if f_no != field_number: continue if self.verbose: print(' Analyzing field: %d' % f_no) self.field_names[f_no] = miscer.get_field_name( self.filename, self.dialect, f_no) if max_freq_number is None: if field_number is None: max_items = MAX_FREQ_MULTI_COL_DEFAULT else: max_items = MAX_FREQ_SINGLE_COL_DEFAULT else: max_items = max_freq_number (self.field_freqs[f_no], self.field_trunc[f_no], self.field_rows_invalid[f_no]) = miscer.get_field_freq( self.filename, self.dialect, f_no, max_items, read_limit) field_freqs = list(self.field_freqs[f_no].items()) self.field_types[f_no] = typer.get_field_type( self.field_freqs[f_no]) if field_types_overrides: for col_no in field_types_overrides: self.field_types[col_no] = field_types_overrides[col_no] self.field_max[f_no] = miscer.get_max(self.field_types[f_no], field_freqs) self.field_min[f_no] = miscer.get_min(self.field_types[f_no], field_freqs) if self.field_types[f_no] == 'string': self.field_case[f_no] = miscer.get_case( self.field_types[f_no], field_freqs) self.field_min_length[f_no] = miscer.get_min_length( field_freqs) self.field_max_length[f_no] = miscer.get_max_length( field_freqs) self.field_mean_length[f_no] = mather.get_mean_length( field_freqs) else: self.field_case[f_no] = None self.field_min_length[f_no] = None self.field_max_length[f_no] = None self.field_mean_length[f_no] = None if self.field_types[f_no] in ('integer', 'float'): self.field_mean[f_no] = mather.get_mean(field_freqs) self.field_median[f_no] = mather.get_median(field_freqs) (self.variance[f_no], self.stddev[f_no]) \ = mather.get_variance_and_stddev(field_freqs, self.field_mean[f_no]) else: self.field_mean[f_no] = None self.field_median[f_no] = None self.variance[f_no] = None self.stddev[f_no] = None
def test_math_empty_list(self): list_1 = [] list_1a = (None, None) assert mod.get_variance_and_stddev(list_1) == list_1a
def test_math_ignoring_bad_value(self): list_1 = [(2, 2), (3, 'foo')] list_1a = (0.0, 0.0) assert mod.get_variance_and_stddev(list_1, 2) == list_1a assert mod.get_variance_and_stddev(list_1) == list_1a