def analyze_fields(self, field_number: Optional[int] = None, field_types_overrides: Optional[Dict[int, str]] = None, max_freq_number: Optional[int] = None, read_limit: int = -1) -> None: """ Determines types, names, and characteristics of fields. Arguments: - field_number: if None, then analyzes all fields, otherwise analyzes just the single field (based on zero-offset) - field_types_overrides: - max_freq_number: limits size of collected frequency distribution, allowing for faster analysis or analysis of very large high-cardinality fields. - read_limit: a performance setting that stops file reads after this number. The default is -1 which means 'no limit'. Returns: - Nothing directly - populates instance variables. """ self.max_freq_number = max_freq_number if self.verbose: print('Field Analysis Progress: ') for f_no in range(self.field_cnt): if field_number is not None: # optional analysis of a single field if f_no != field_number: continue if self.verbose: print(' Analyzing field: %d' % f_no) self.field_names[f_no] = miscer.get_field_name(self.filename, self.dialect, f_no) if max_freq_number is None: if field_number is None: max_items = MAX_FREQ_MULTI_COL_DEFAULT else: max_items = MAX_FREQ_SINGLE_COL_DEFAULT else: max_items = max_freq_number (self.field_freqs[f_no], self.field_trunc[f_no], self.field_rows_invalid[f_no]) = miscer.get_field_freq(self.filename, self.dialect, f_no, max_items, read_limit) field_freqs = list(self.field_freqs[f_no].items()) self.field_types[f_no] = typer.get_field_type(self.field_freqs[f_no]) if field_types_overrides: for col_no in field_types_overrides: self.field_types[col_no] = field_types_overrides[col_no] self.field_max[f_no] = miscer.get_max(self.field_types[f_no], field_freqs) self.field_min[f_no] = miscer.get_min(self.field_types[f_no], field_freqs) if self.field_types[f_no] == 'string': self.field_case[f_no] = miscer.get_case(self.field_types[f_no], field_freqs) self.field_min_length[f_no] = miscer.get_min_length(field_freqs) self.field_max_length[f_no] = miscer.get_max_length(field_freqs) self.field_mean_length[f_no] = mather.get_mean_length(field_freqs) else: self.field_case[f_no] = None self.field_min_length[f_no] = None self.field_max_length[f_no] = None self.field_mean_length[f_no] = None if self.field_types[f_no] in ('integer', 'float'): self.field_mean[f_no] = mather.get_mean(field_freqs) self.field_median[f_no] = mather.get_median(field_freqs) (self.variance[f_no], self.stddev[f_no]) \ = mather.get_variance_and_stddev(field_freqs, self.field_mean[f_no]) else: self.field_mean[f_no] = None self.field_median[f_no] = None self.variance[f_no] = None self.stddev[f_no] = None
def test_easy_multiples(self): assert mod.get_mean_length([('abc', 99), ('abcdef', 1)]) == 3.03
def test_list_of_tuples(self): assert mod.get_mean_length([('abc', 99), ('abcdef', 1)]) == 3.03
def test_unknowns(self): assert mod.get_mean_length([('UNK', 1), ('unknown', 3), (' ', 99), ('', 99), ('a', 4), ('b', 2)]) == 1
def test_easy_singles(self): assert mod.get_mean_length([('abc', 1), ('abcdefg', 1)]) == 5
def test_empty(self): assert mod.get_mean_length([]) == 0
def test_none(self): assert mod.get_mean_length(None) == 0
def analyze_fields(self, field_number: Optional[int] = None, field_types_overrides: Optional[Dict[int, str]] = None, max_freq_number: Optional[int] = None, read_limit: int = -1) -> None: """ Determines types, names, and characteristics of fields. Arguments: - field_number: if None, then analyzes all fields, otherwise analyzes just the single field (based on zero-offset) - field_types_overrides: - max_freq_number: limits size of collected frequency distribution, allowing for faster analysis or analysis of very large high-cardinality fields. - read_limit: a performance setting that stops file reads after this number. The default is -1 which means 'no limit'. Returns: - Nothing directly - populates instance variables. """ self.max_freq_number = max_freq_number if self.verbose: print('Field Analysis Progress: ') for f_no in range(self.field_cnt): if field_number is not None: # optional analysis of a single field if f_no != field_number: continue if self.verbose: print(' Analyzing field: %d' % f_no) self.field_names[f_no] = miscer.get_field_name( self.filename, self.dialect, f_no) if max_freq_number is None: if field_number is None: max_items = MAX_FREQ_MULTI_COL_DEFAULT else: max_items = MAX_FREQ_SINGLE_COL_DEFAULT else: max_items = max_freq_number (self.field_freqs[f_no], self.field_trunc[f_no], self.field_rows_invalid[f_no]) = miscer.get_field_freq( self.filename, self.dialect, f_no, max_items, read_limit) field_freqs = list(self.field_freqs[f_no].items()) self.field_types[f_no] = typer.get_field_type( self.field_freqs[f_no]) if field_types_overrides: for col_no in field_types_overrides: self.field_types[col_no] = field_types_overrides[col_no] self.field_max[f_no] = miscer.get_max(self.field_types[f_no], field_freqs) self.field_min[f_no] = miscer.get_min(self.field_types[f_no], field_freqs) if self.field_types[f_no] == 'string': self.field_case[f_no] = miscer.get_case( self.field_types[f_no], field_freqs) self.field_min_length[f_no] = miscer.get_min_length( field_freqs) self.field_max_length[f_no] = miscer.get_max_length( field_freqs) self.field_mean_length[f_no] = mather.get_mean_length( field_freqs) else: self.field_case[f_no] = None self.field_min_length[f_no] = None self.field_max_length[f_no] = None self.field_mean_length[f_no] = None if self.field_types[f_no] in ('integer', 'float'): self.field_mean[f_no] = mather.get_mean(field_freqs) self.field_median[f_no] = mather.get_median(field_freqs) (self.variance[f_no], self.stddev[f_no]) \ = mather.get_variance_and_stddev(field_freqs, self.field_mean[f_no]) else: self.field_mean[f_no] = None self.field_median[f_no] = None self.variance[f_no] = None self.stddev[f_no] = None