def nested_data_to_arrays( data: Sequence, columns: Index | None, index: Index | None, dtype: DtypeObj | None, ) -> tuple[list[ArrayLike], Index, Index]: """ Convert a single sequence of arrays to multiple arrays. """ # By the time we get here we have already checked treat_as_nested(data) if is_named_tuple(data[0]) and columns is None: columns = ensure_index(data[0]._fields) arrays, columns = to_arrays(data, columns, dtype=dtype) columns = ensure_index(columns) if index is None: if isinstance(data[0], ABCSeries): index = _get_names_from_index(data) elif isinstance(data[0], Categorical): # GH#38845 hit in test_constructor_categorical index = default_index(len(data[0])) else: index = default_index(len(data)) return arrays, columns, index
def _get_concat_axis(self) -> Index: """ Return index to be used along concatenation axis. """ if self._is_series: if self.bm_axis == 0: indexes = [x.index for x in self.objs] elif self.ignore_index: idx = default_index(len(self.objs)) return idx elif self.keys is None: names: list[Hashable] = [None] * len(self.objs) num = 0 has_names = False for i, x in enumerate(self.objs): if not isinstance(x, ABCSeries): raise TypeError( f"Cannot concatenate type 'Series' with " f"object of type '{type(x).__name__}'" ) if x.name is not None: names[i] = x.name has_names = True else: names[i] = num num += 1 if has_names: return Index(names) else: return default_index(len(self.objs)) else: return ensure_index(self.keys).set_names(self.names) else: indexes = [x.axes[self.axis] for x in self.objs] if self.ignore_index: idx = default_index(sum(len(i) for i in indexes)) return idx if self.keys is None: concat_axis = _concat_indexes(indexes) else: concat_axis = _make_concat_multiindex( indexes, self.keys, self.levels, self.names ) self._maybe_check_integrity(concat_axis) return concat_axis
def _get_axes(N: int, K: int, index: Index | None, columns: Index | None) -> tuple[Index, Index]: # helper to create the axes as indexes # return axes or defaults if index is None: index = default_index(N) else: index = ensure_index(index) if columns is None: columns = default_index(K) else: columns = ensure_index(columns) return index, columns
def _list_of_series_to_arrays( data: list, columns: Index | None, ) -> tuple[np.ndarray, Index]: # returned np.ndarray has ndim == 2 if columns is None: # We know pass_data is non-empty because data[0] is a Series pass_data = [x for x in data if isinstance(x, (ABCSeries, ABCDataFrame))] columns = get_objs_combined_axis(pass_data, sort=False) indexer_cache: dict[int, np.ndarray] = {} aligned_values = [] for s in data: index = getattr(s, "index", None) if index is None: index = default_index(len(s)) if id(index) in indexer_cache: indexer = indexer_cache[id(index)] else: indexer = indexer_cache[id(index)] = index.get_indexer(columns) values = extract_array(s, extract_numpy=True) aligned_values.append(algorithms.take_nd(values, indexer)) # error: Argument 1 to "vstack" has incompatible type "List[ExtensionArray]"; # expected "Sequence[Union[Union[int, float, complex, str, bytes, generic], # Sequence[Union[int, float, complex, str, bytes, generic]], # Sequence[Sequence[Any]], _SupportsArray]]" content = np.vstack(aligned_values) # type: ignore[arg-type] return content, columns
def _list_of_series_to_arrays( data: list, columns: Index | None, ) -> tuple[np.ndarray, Index]: # returned np.ndarray has ndim == 2 if columns is None: # We know pass_data is non-empty because data[0] is a Series pass_data = [ x for x in data if isinstance(x, (ABCSeries, ABCDataFrame)) ] columns = get_objs_combined_axis(pass_data, sort=False) indexer_cache: dict[int, np.ndarray] = {} aligned_values = [] for s in data: index = getattr(s, "index", None) if index is None: index = default_index(len(s)) if id(index) in indexer_cache: indexer = indexer_cache[id(index)] else: indexer = indexer_cache[id(index)] = index.get_indexer(columns) values = extract_array(s, extract_numpy=True) aligned_values.append(algorithms.take_nd(values, indexer)) content = np.vstack(aligned_values) return content, columns
def _validate_or_indexify_columns( content: list[np.ndarray], columns: Index | None ) -> Index: """ If columns is None, make numbers as column names; Otherwise, validate that columns have valid length. Parameters ---------- content : list of np.ndarrays columns : Index or None Returns ------- Index If columns is None, assign positional column index value as columns. Raises ------ 1. AssertionError when content is not composed of list of lists, and if length of columns is not equal to length of content. 2. ValueError when content is list of lists, but length of each sub-list is not equal 3. ValueError when content is list of lists, but length of sub-list is not equal to length of content """ if columns is None: columns = default_index(len(content)) else: # Add mask for data which is composed of list of lists is_mi_list = isinstance(columns, list) and all( isinstance(col, list) for col in columns ) if not is_mi_list and len(columns) != len(content): # pragma: no cover # caller's responsibility to check for this... raise AssertionError( f"{len(columns)} columns passed, passed data had " f"{len(content)} columns" ) elif is_mi_list: # check if nested list column, length of each sub-list should be equal if len({len(col) for col in columns}) > 1: raise ValueError( "Length of columns passed for MultiIndex columns is different" ) # if columns is not empty and length of sublist is not equal to content elif columns and len(columns[0]) != len(content): raise ValueError( f"{len(columns[0])} columns passed, passed data had " f"{len(content)} columns" ) return columns
def grouped_reduce(self, func, ignore_failures: bool = False): """ ignore_failures : bool, default False Not used; for compatibility with ArrayManager/BlockManager. """ arr = self.array res = func(arr) index = default_index(len(res)) mgr = type(self).from_array(res, index) return mgr
def _prep_index(data, index, columns): from pandas.core.indexes.api import ( default_index, ensure_index, ) N, K = data.shape if index is None: index = default_index(N) else: index = ensure_index(index) if columns is None: columns = default_index(K) else: columns = ensure_index(columns) if len(columns) != K: raise ValueError(f"Column length mismatch: {len(columns)} vs. {K}") if len(index) != N: raise ValueError(f"Index length mismatch: {len(index)} vs. {N}") return index, columns
def rec_array_to_mgr( data: MaskedRecords | np.recarray | np.ndarray, index, columns, dtype: DtypeObj | None, copy: bool, typ: str, ): """ Extract from a masked rec array and create the manager. """ # essentially process a record array then fill it fdata = ma.getdata(data) if index is None: index = default_index(len(fdata)) else: index = ensure_index(index) if columns is not None: columns = ensure_index(columns) arrays, arr_columns = to_arrays(fdata, columns) # fill if needed if isinstance(data, np.ma.MaskedArray): # GH#42200 we only get here with MaskedRecords, but check for the # parent class MaskedArray to avoid the need to import MaskedRecords data = cast("MaskedRecords", data) new_arrays = fill_masked_arrays(data, arr_columns) else: # error: Incompatible types in assignment (expression has type # "List[ExtensionArray]", variable has type "List[ndarray]") new_arrays = arrays # type: ignore[assignment] # create the manager # error: Argument 1 to "reorder_arrays" has incompatible type "List[ndarray]"; # expected "List[Union[ExtensionArray, ndarray]]" arrays, arr_columns = reorder_arrays( new_arrays, arr_columns, columns, len(index) # type: ignore[arg-type] ) if columns is None: columns = arr_columns mgr = arrays_to_mgr(arrays, columns, index, dtype=dtype, typ=typ) if copy: mgr = mgr.copy() return mgr
def _get_names_from_index(data) -> Index: has_some_name = any(getattr(s, "name", None) is not None for s in data) if not has_some_name: return default_index(len(data)) index: list[Hashable] = list(range(len(data))) count = 0 for i, s in enumerate(data): n = getattr(s, "name", None) if n is not None: index[i] = n else: index[i] = f"Unnamed {count}" count += 1 return Index(index)
def to_arrays(data, columns: Index | None, dtype: DtypeObj | None = None) -> tuple[list[ArrayLike], Index]: """ Return list of arrays, columns. Returns ------- list[ArrayLike] These will become columns in a DataFrame. Index This will become frame.columns. Notes ----- Ensures that len(result_arrays) == len(result_index). """ if isinstance(data, ABCDataFrame): # see test_from_records_with_index_data, test_from_records_bad_index_column if columns is not None: arrays = [ data._ixs(i, axis=1).values for i, col in enumerate(data.columns) if col in columns ] else: columns = data.columns arrays = [data._ixs(i, axis=1).values for i in range(len(columns))] return arrays, columns if not len(data): if isinstance(data, np.ndarray): if data.dtype.names is not None: # i.e. numpy structured array columns = ensure_index(data.dtype.names) arrays = [data[name] for name in columns] if len(data) == 0: # GH#42456 the indexing above results in list of 2D ndarrays # TODO: is that an issue with numpy? for i, arr in enumerate(arrays): if arr.ndim == 2: arrays[i] = arr[:, 0] return arrays, columns return [], ensure_index([]) elif isinstance(data[0], Categorical): # GH#38845 deprecate special case warnings.warn( "The behavior of DataFrame([categorical, ...]) is deprecated and " "in a future version will be changed to match the behavior of " "DataFrame([any_listlike, ...]). " "To retain the old behavior, pass as a dictionary " "DataFrame({col: categorical, ..})", FutureWarning, stacklevel=find_stack_level(), ) if columns is None: columns = default_index(len(data)) elif len(columns) > len(data): raise ValueError("len(columns) > len(data)") elif len(columns) < len(data): # doing this here is akin to a pre-emptive reindex data = data[:len(columns)] return data, columns elif isinstance(data, np.ndarray) and data.dtype.names is not None: # e.g. recarray columns = Index(list(data.dtype.names)) arrays = [data[k] for k in columns] return arrays, columns if isinstance(data[0], (list, tuple)): arr = _list_to_arrays(data) elif isinstance(data[0], abc.Mapping): arr, columns = _list_of_dict_to_arrays(data, columns) elif isinstance(data[0], ABCSeries): arr, columns = _list_of_series_to_arrays(data, columns) else: # last ditch effort data = [tuple(x) for x in data] arr = _list_to_arrays(data) content, columns = _finalize_columns_and_data(arr, columns, dtype) return content, columns
def _extract_index(data) -> Index: """ Try to infer an Index from the passed data, raise ValueError on failure. """ index = None if len(data) == 0: index = Index([]) else: raw_lengths = [] indexes: list[list[Hashable] | Index] = [] have_raw_arrays = False have_series = False have_dicts = False for val in data: if isinstance(val, ABCSeries): have_series = True indexes.append(val.index) elif isinstance(val, dict): have_dicts = True indexes.append(list(val.keys())) elif is_list_like(val) and getattr(val, "ndim", 1) == 1: have_raw_arrays = True raw_lengths.append(len(val)) elif isinstance(val, np.ndarray) and val.ndim > 1: raise ValueError( "Per-column arrays must each be 1-dimensional") if not indexes and not raw_lengths: raise ValueError( "If using all scalar values, you must pass an index") elif have_series: index = union_indexes(indexes) elif have_dicts: index = union_indexes(indexes, sort=False) if have_raw_arrays: lengths = list(set(raw_lengths)) if len(lengths) > 1: raise ValueError("All arrays must be of the same length") if have_dicts: raise ValueError( "Mixing dicts with non-Series may lead to ambiguous ordering." ) if have_series: assert index is not None # for mypy if lengths[0] != len(index): msg = (f"array length {lengths[0]} does not match index " f"length {len(index)}") raise ValueError(msg) else: index = default_index(lengths[0]) # error: Argument 1 to "ensure_index" has incompatible type "Optional[Index]"; # expected "Union[Union[Union[ExtensionArray, ndarray], Index, Series], # Sequence[Any]]" return ensure_index(index) # type: ignore[arg-type]