def reorder_values_array(arr, variables): for col, var in enumerate(variables): if getattr(var, "fix_order", False) and len(var.values) < 1000: new_order = var.ordered_values(var.values) if new_order == var.values: continue arr[:, col] += 1000 for i, val in enumerate(var.values): bn.replace(arr[:, col], 1000 + i, new_order.index(val)) var.values = new_order delattr(var, "fix_order")
def test_replace_nan_int(): "Test replace, int array, old=nan, new=0" a = np.arange(2 * 3 * 4).reshape(2, 3, 4) actual = a.copy() bn.replace(actual, np.nan, 0) desired = a.copy() msg = 'replace failed on int input looking for nans' assert_array_equal(actual, desired, err_msg=msg) actual = a.copy() bn.slow.replace(actual, np.nan, 0) msg = 'slow.replace failed on int input looking for nans' assert_array_equal(actual, desired, err_msg=msg)
def test_replace_nan_int(): "Test replace, int array, old=nan, new=0" a = np.arange(2*3*4).reshape(2,3,4) actual = a.copy() bn.replace(actual, np.nan, 0) desired = a.copy() msg = 'replace failed on int input looking for nans' assert_array_equal(actual, desired, err_msg=msg) actual = a.copy() bn.slow.replace(actual, np.nan, 0) msg = 'slow.replace failed on int input looking for nans' assert_array_equal(actual, desired, err_msg=msg)
def reorder_values_array(self, arr, variables): newvars = [] for col, var in enumerate(variables): if getattr(var, "fix_order", False): nvar = var.make(var.name, var.values, var.ordered) nvar.attributes = var.attributes move = len(var.values) if nvar.values != var.values: arr[:, col] += move for i, val in enumerate(var.values): bn.replace(arr[:, col], move + i, nvar.values.index(val)) var = nvar newvars.append(var) return newvars
def data_table(self, data, headers=None): """ Return Orange.data.Table given rows of `headers` (iterable of iterable) and rows of `data` (iterable of iterable; if ``numpy.ndarray``, might as well **have it sorted column-major**, e.g. ``order='F'``). Basically, the idea of subclasses is to produce those two iterables, however they might. If `headers` is not provided, the header rows are extracted from `data`, assuming they precede it. """ if not headers: headers, data = self.parse_headers(data) # Consider various header types (single-row, two-row, three-row, none) if 3 == len(headers): names, types, flags = map(list, headers) else: if 1 == len(headers): HEADER1_FLAG_SEP = '#' # First row format either: # 1) delimited column names # 2) -||- with type and flags prepended, separated by #, # e.g. d#sex,c#age,cC#IQ _flags, names = zip(*[ i.split(HEADER1_FLAG_SEP, 1) if HEADER1_FLAG_SEP in i else ('', i) for i in headers[0] ]) names = list(names) elif 2 == len(headers): names, _flags = map(list, headers) else: # Use heuristics for everything names, _flags = [], [] types = [ ''.join(filter(str.isupper, flag)).lower() for flag in _flags ] flags = [Flags.join(filter(str.islower, flag)) for flag in _flags] # Determine maximum row length rowlen = max(map(len, (names, types, flags))) def _equal_length(lst): lst.extend([''] * (rowlen - len(lst))) return lst # Ensure all data is of equal width in a column-contiguous array data = np.array([_equal_length(list(row)) for row in data if any(row)], copy=False, dtype=object, order='F') # Data may actually be longer than headers were try: rowlen = data.shape[1] except IndexError: pass else: for lst in (names, types, flags): _equal_length(lst) NAMEGEN = namegen('Feature ', 1) Xcols, attrs = [], [] Mcols, metas = [], [] Ycols, clses = [], [] Wcols = [] # Iterate through the columns for col in range(rowlen): flag = Flags(Flags.split(flags[col])) if flag.i: continue type_flag = types and types[col].strip() try: orig_values = [ np.nan if i in MISSING_VALUES else i for i in (i.strip() for i in data[:, col]) ] except IndexError: # No data instances leads here orig_values = [] # In this case, coltype could be anything. It's set as-is # only to satisfy test_table.TableTestCase.test_append coltype = DiscreteVariable coltype_kwargs = {} valuemap = [] values = orig_values if type_flag in StringVariable.TYPE_HEADERS: coltype = StringVariable elif type_flag in ContinuousVariable.TYPE_HEADERS: coltype = ContinuousVariable try: values = [float(i) for i in orig_values] except ValueError: for row, num in enumerate(orig_values): try: float(num) except ValueError: break raise ValueError('Non-continuous value in (1-based) ' 'line {}, column {}'.format( row + len(headers) + 1, col + 1)) elif type_flag in TimeVariable.TYPE_HEADERS: coltype = TimeVariable elif (type_flag in DiscreteVariable.TYPE_HEADERS or _RE_DISCRETE_LIST.match(type_flag)): if _RE_DISCRETE_LIST.match(type_flag): valuemap = Flags.split(type_flag) coltype_kwargs.update(ordered=True) else: valuemap = sorted(set(orig_values) - {np.nan}) else: # No known type specified, use heuristics is_discrete = is_discrete_values(orig_values) if is_discrete: valuemap = sorted(is_discrete) else: try: values = [float(i) for i in orig_values] except ValueError: tvar = TimeVariable('_') try: values = [tvar.parse(i) for i in orig_values] except ValueError: coltype = StringVariable else: coltype = TimeVariable else: coltype = ContinuousVariable if valuemap: # Map discrete data to ints def valuemap_index(val): try: return valuemap.index(val) except ValueError: return np.nan values = np.vectorize(valuemap_index, otypes=[float])(orig_values) coltype = DiscreteVariable coltype_kwargs.update(values=valuemap) if coltype is StringVariable: values = ['' if i is np.nan else i for i in orig_values] if flag.m or coltype is StringVariable: append_to = (Mcols, metas) elif flag.w: append_to = (Wcols, None) elif flag.c: append_to = (Ycols, clses) else: append_to = (Xcols, attrs) cols, domain_vars = append_to cols.append(col) if domain_vars is not None: if names and names[col]: # Use existing variable if available var = coltype.make(names[col].strip(), **coltype_kwargs) else: # Never use existing for un-named variables var = coltype(next(NAMEGEN), **coltype_kwargs) var.attributes.update(flag.attributes) domain_vars.append(var) # Reorder discrete values to match existing variable if var.is_discrete and not var.ordered: new_order, old_order = var.values, coltype_kwargs.get( 'values', var.values) if new_order != old_order: offset = len(new_order) column = values if data.ndim > 1 else data column += offset for i, val in enumerate(var.values): try: oldval = old_order.index(val) except ValueError: continue bn.replace(column, offset + oldval, new_order.index(val)) if coltype is TimeVariable: # Re-parse the values because only now after coltype.make call # above, variable var is the correct one values = [var.parse(i) for i in orig_values] # Write back the changed data. This is needeed to pass the # correct, converted values into Table.from_numpy below try: data[:, col] = values except IndexError: pass domain = Domain(attrs, clses, metas) if not data.size: return Table.from_domain(domain, 0) table = Table.from_numpy(domain, data[:, Xcols].astype(float, order='C'), data[:, Ycols].astype(float, order='C'), data[:, Mcols].astype(object, order='C'), data[:, Wcols].astype(float, order='C')) return table
def data_table(self, data, headers=None): """ Return Orange.data.Table given rows of `headers` (iterable of iterable) and rows of `data` (iterable of iterable; if ``numpy.ndarray``, might as well **have it sorted column-major**, e.g. ``order='F'``). Basically, the idea of subclasses is to produce those two iterables, however they might. If `headers` is not provided, the header rows are extracted from `data`, assuming they precede it. """ if not headers: headers, data = self.parse_headers(data) # Consider various header types (single-row, two-row, three-row, none) if 3 == len(headers): names, types, flags = map(list, headers) else: if 1 == len(headers): HEADER1_FLAG_SEP = '#' # First row format either: # 1) delimited column names # 2) -||- with type and flags prepended, separated by #, # e.g. d#sex,c#age,cC#IQ _flags, names = zip(*[i.split(HEADER1_FLAG_SEP, 1) if HEADER1_FLAG_SEP in i else ('', i) for i in headers[0]]) names = list(names) elif 2 == len(headers): names, _flags = map(list, headers) else: # Use heuristics for everything names, _flags = [], [] types = [''.join(filter(str.isupper, flag)).lower() for flag in _flags] flags = [Flags.join(filter(str.islower, flag)) for flag in _flags] # Determine maximum row length rowlen = max(map(len, (names, types, flags))) def _equal_length(lst): lst.extend(['']*(rowlen - len(lst))) return lst # Ensure all data is of equal width in a column-contiguous array data = np.array([_equal_length(list(row)) for row in data if any(row)], copy=False, dtype=object, order='F') # Data may actually be longer than headers were try: rowlen = data.shape[1] except IndexError: pass else: for lst in (names, types, flags): _equal_length(lst) NAMEGEN = namegen('Feature ', 1) Xcols, attrs = [], [] Mcols, metas = [], [] Ycols, clses = [], [] Wcols = [] # Iterate through the columns for col in range(rowlen): flag = Flags(Flags.split(flags[col])) if flag.i: continue type_flag = types and types[col].strip() try: orig_values = [np.nan if i in MISSING_VALUES else i for i in (i.strip() for i in data[:, col])] except IndexError: # No data instances leads here orig_values = [] # In this case, coltype could be anything. It's set as-is # only to satisfy test_table.TableTestCase.test_append coltype = DiscreteVariable coltype_kwargs = {} valuemap = [] values = orig_values if type_flag in StringVariable.TYPE_HEADERS: coltype = StringVariable elif type_flag in ContinuousVariable.TYPE_HEADERS: coltype = ContinuousVariable try: values = [float(i) for i in orig_values] except ValueError: for row, num in enumerate(orig_values): try: float(num) except ValueError: break raise ValueError('Non-continuous value in (1-based) ' 'line {}, column {}'.format(row + len(headers) + 1, col + 1)) elif type_flag in TimeVariable.TYPE_HEADERS: coltype = TimeVariable elif (type_flag in DiscreteVariable.TYPE_HEADERS or _RE_DISCRETE_LIST.match(type_flag)): if _RE_DISCRETE_LIST.match(type_flag): valuemap = Flags.split(type_flag) coltype_kwargs.update(ordered=True) else: valuemap = sorted(set(orig_values) - {np.nan}) else: # No known type specified, use heuristics is_discrete = is_discrete_values(orig_values) if is_discrete: valuemap = sorted(is_discrete) else: try: values = [float(i) for i in orig_values] except ValueError: tvar = TimeVariable('_') try: values = [tvar.parse(i) for i in orig_values] except ValueError: coltype = StringVariable else: coltype = TimeVariable else: coltype = ContinuousVariable if valuemap: # Map discrete data to ints def valuemap_index(val): try: return valuemap.index(val) except ValueError: return np.nan values = np.vectorize(valuemap_index, otypes=[float])(orig_values) coltype = DiscreteVariable coltype_kwargs.update(values=valuemap) if coltype is StringVariable: values = ['' if i is np.nan else i for i in orig_values] if flag.m or coltype is StringVariable: append_to = (Mcols, metas) elif flag.w: append_to = (Wcols, None) elif flag.c: append_to = (Ycols, clses) else: append_to = (Xcols, attrs) cols, domain_vars = append_to cols.append(col) if domain_vars is not None: if names and names[col]: # Use existing variable if available var = coltype.make(names[col].strip(), **coltype_kwargs) else: # Never use existing for un-named variables var = coltype(next(NAMEGEN), **coltype_kwargs) var.attributes.update(flag.attributes) domain_vars.append(var) # Reorder discrete values to match existing variable if var.is_discrete and not var.ordered: new_order, old_order = var.values, coltype_kwargs.get('values', var.values) if new_order != old_order: offset = len(new_order) column = values if data.ndim > 1 else data column += offset for i, val in enumerate(var.values): try: oldval = old_order.index(val) except ValueError: continue bn.replace(column, offset + oldval, new_order.index(val)) if coltype is TimeVariable: # Re-parse the values because only now after coltype.make call # above, variable var is the correct one values = [var.parse(i) for i in orig_values] # Write back the changed data. This is needeed to pass the # correct, converted values into Table.from_numpy below try: data[:, col] = values except IndexError: pass from Orange.data import Table, Domain domain = Domain(attrs, clses, metas) if not data.size: return Table.from_domain(domain, 0) table = Table.from_numpy(domain, data[:, Xcols].astype(float, order='C'), data[:, Ycols].astype(float, order='C'), data[:, Mcols].astype(object, order='C'), data[:, Wcols].astype(float, order='C')) return table