def _build_design_matrix(design_info, factor_info_to_values, dtype):
factor_to_values = {}
need_reshape = False
num_rows = None
for factor_info, value in six.iteritems(factor_info_to_values):
# It's possible that the same factor appears in multiple different
# FactorInfo objects (e.g. if someone is simultaneously building two
# DesignInfo objects that started out as part of different
# formulas). Skip any factor_info that is not our expected
# factor_info.
if design_info.factor_infos.get(factor_info.factor) is not factor_info:
continue
factor_to_values[factor_info.factor] = value
if num_rows is not None:
assert num_rows == value.shape[0]
else:
num_rows = value.shape[0]
if num_rows is None:
# We have no dependence on the data -- e.g. an empty termlist, or
# only an intercept term.
num_rows = 1
need_reshape = True
shape = (num_rows, len(design_info.column_names))
m = DesignMatrix(np.empty(shape, dtype=dtype), design_info)
start_column = 0
for term, subterms in six.iteritems(design_info.term_codings):
for subterm in subterms:
end_column = start_column + subterm.num_columns
m_slice = m[:, start_column:end_column]
_build_subterm(subterm, design_info.factor_infos,
factor_to_values, m_slice)
start_column = end_column
assert start_column == m.shape[1]
return need_reshape, m
def _build(self, evaluator_to_values, dtype):
factor_to_values = {}
need_reshape = False
num_rows = None
for evaluator, value in evaluator_to_values.iteritems():
if evaluator in self._evaluators:
factor_to_values[evaluator.factor] = value
if num_rows is not None:
assert num_rows == value.shape[0]
else:
num_rows = value.shape[0]
if num_rows is None:
# We have no dependence on the data -- e.g. an empty termlist, or
# only an intercept term.
num_rows = 1
need_reshape = True
m = DesignMatrix(np.empty((num_rows, self.total_columns), dtype=dtype),
self.design_info)
start_column = 0
for term in self._termlist:
for column_builder in self._term_to_column_builders[term]:
end_column = start_column + column_builder.total_columns
m_slice = m[:, start_column:end_column]
column_builder.build(factor_to_values, m_slice)
start_column = end_column
assert start_column == self.total_columns
return need_reshape, m
def _regularize_matrix(m, default_column_prefix):
di = DesignInfo.from_array(m, default_column_prefix)
if have_pandas and isinstance(m,
(pandas.Series, pandas.DataFrame)):
orig_index = m.index
else:
orig_index = None
if return_type == "dataframe":
m = atleast_2d_column_default(m, preserve_pandas=True)
m = pandas.DataFrame(m)
m.columns = di.column_names
m.design_info = di
return (m, orig_index)
else:
return (DesignMatrix(m, di), orig_index)
def test_formula_likes():
# Plain array-like, rhs only
t([[1, 2, 3], [4, 5, 6]], {}, 0, False, [[1, 2, 3], [4, 5, 6]],
["x0", "x1", "x2"])
t((None, [[1, 2, 3], [4, 5, 6]]), {}, 0, False, [[1, 2, 3], [4, 5, 6]],
["x0", "x1", "x2"])
t(np.asarray([[1, 2, 3], [4, 5, 6]]), {}, 0, False, [[1, 2, 3], [4, 5, 6]],
["x0", "x1", "x2"])
t((None, np.asarray([[1, 2, 3], [4, 5, 6]])), {}, 0, False,
[[1, 2, 3], [4, 5, 6]], ["x0", "x1", "x2"])
dm = DesignMatrix([[1, 2, 3], [4, 5, 6]], default_column_prefix="foo")
t(dm, {}, 0, False, [[1, 2, 3], [4, 5, 6]], ["foo0", "foo1", "foo2"])
t((None, dm), {}, 0, False, [[1, 2, 3], [4, 5, 6]],
["foo0", "foo1", "foo2"])
# Plain array-likes, lhs and rhs
t(([1, 2], [[1, 2, 3], [4, 5, 6]]), {}, 0, False, [[1, 2, 3], [4, 5, 6]],
["x0", "x1", "x2"], [[1], [2]], ["y0"])
t(([[1], [2]], [[1, 2, 3], [4, 5, 6]]), {}, 0, False,
[[1, 2, 3], [4, 5, 6]], ["x0", "x1", "x2"], [[1], [2]], ["y0"])
t((np.asarray([1, 2]), np.asarray([[1, 2, 3], [4, 5, 6]])), {}, 0, False,
[[1, 2, 3], [4, 5, 6]], ["x0", "x1", "x2"], [[1], [2]], ["y0"])
t((np.asarray([[1], [2]]), np.asarray([[1, 2, 3], [4, 5, 6]])), {}, 0,
False, [[1, 2, 3], [4, 5, 6]], ["x0", "x1", "x2"], [[1], [2]], ["y0"])
x_dm = DesignMatrix([[1, 2, 3], [4, 5, 6]], default_column_prefix="foo")
y_dm = DesignMatrix([1, 2], default_column_prefix="bar")
t((y_dm, x_dm), {}, 0, False, [[1, 2, 3], [4, 5, 6]],
["foo0", "foo1", "foo2"], [[1], [2]], ["bar0"])
# number of rows must match
t_invalid(([1, 2, 3], [[1, 2, 3], [4, 5, 6]]), {}, 0)
# tuples must have the right size
t_invalid(([[1, 2, 3]], ), {}, 0)
t_invalid(([[1, 2, 3]], [[1, 2, 3]], [[1, 2, 3]]), {}, 0)
# plain Series and DataFrames
if have_pandas:
# Names are extracted
t(pandas.DataFrame({"x": [1, 2, 3]}), {}, 0, False, [[1], [2], [3]],
["x"])
t(pandas.Series([1, 2, 3], name="asdf"), {}, 0, False, [[1], [2], [3]],
["asdf"])
t((pandas.DataFrame({"y": [4, 5, 6]
}), pandas.DataFrame({"x": [1, 2, 3]})), {}, 0,
False, [[1], [2], [3]], ["x"], [[4], [5], [6]], ["y"])
t((pandas.Series([4, 5, 6],
name="y"), pandas.Series([1, 2, 3], name="x")), {}, 0,
False, [[1], [2], [3]], ["x"], [[4], [5], [6]], ["y"])
# Or invented
t((pandas.DataFrame([[4, 5, 6]]),
pandas.DataFrame([[1, 2, 3]], columns=[7, 8, 9])), {}, 0, False,
[[1, 2, 3]], ["x7", "x8", "x9"], [[4, 5, 6]], ["y0", "y1", "y2"])
t(pandas.Series([1, 2, 3]), {}, 0, False, [[1], [2], [3]], ["x0"])
# indices must match
t_invalid((pandas.DataFrame(
[[1]], index=[1]), pandas.DataFrame([[1]], index=[2])), {}, 0)
# Foreign ModelDesc factories
class ForeignModelSource(object):
def __patsy_get_model_desc__(self, data):
return ModelDesc([Term([LookupFactor("Y")])],
[Term([LookupFactor("X")])])
foreign_model = ForeignModelSource()
t(foreign_model, {
"Y": [1, 2],
"X": [[1, 2], [3, 4]]
}, 0, True, [[1, 2], [3, 4]], ["X[0]", "X[1]"], [[1], [2]], ["Y"])
class BadForeignModelSource(object):
def __patsy_get_model_desc__(self, data):
return data
t_invalid(BadForeignModelSource(), {}, 0)
# string formulas
t("y ~ x", {
"y": [1, 2],
"x": [3, 4]
}, 0, True, [[1, 3], [1, 4]], ["Intercept", "x"], [[1], [2]], ["y"])
t("~ x", {
"y": [1, 2],
"x": [3, 4]
}, 0, True, [[1, 3], [1, 4]], ["Intercept", "x"])
t("x + y", {
"y": [1, 2],
"x": [3, 4]
}, 0, True, [[1, 3, 1], [1, 4, 2]], ["Intercept", "x", "y"])
# ModelDesc
desc = ModelDesc([], [Term([LookupFactor("x")])])
t(desc, {"x": [1.5, 2.5, 3.5]}, 0, True, [[1.5], [2.5], [3.5]], ["x"])
desc = ModelDesc([], [Term([]), Term([LookupFactor("x")])])
t(desc, {"x": [1.5, 2.5, 3.5]}, 0, True, [[1, 1.5], [1, 2.5], [1, 3.5]],
["Intercept", "x"])
desc = ModelDesc([Term([LookupFactor("y")])],
[Term([]), Term([LookupFactor("x")])])
t(desc, {
"x": [1.5, 2.5, 3.5],
"y": [10, 20, 30]
}, 0, True, [[1, 1.5], [1, 2.5], [1, 3.5]], ["Intercept", "x"],
[[10], [20], [30]], ["y"])
# builders
termlists = (
[],
[Term([LookupFactor("x")])],
[Term([]), Term([LookupFactor("x")])],
)
builders = design_matrix_builders(termlists, lambda: iter([{
"x": [1, 2, 3]
}]))
# twople but with no LHS
t((builders[0], builders[2]), {"x": [10, 20, 30]}, 0, True,
[[1, 10], [1, 20], [1, 30]], ["Intercept", "x"])
# single DesignMatrixBuilder
t(builders[2], {"x": [10, 20, 30]}, 0, True, [[1, 10], [1, 20], [1, 30]],
["Intercept", "x"])
# twople with LHS
t((builders[1], builders[2]), {"x": [10, 20, 30]}, 0, True,
[[1, 10], [1, 20], [1, 30]], ["Intercept", "x"], [[10], [20], [30]],
["x"])
# check depth arguments
x_in_env = [1, 2, 3]
t("~ x_in_env", {}, 0, True, [[1, 1], [1, 2], [1, 3]],
["Intercept", "x_in_env"])
t("~ x_in_env", {"x_in_env": [10, 20, 30]}, 0, True,
[[1, 10], [1, 20], [1, 30]], ["Intercept", "x_in_env"])
# Trying to pull x_in_env out of our *caller* shouldn't work.
t_invalid("~ x_in_env", {}, 1, exc=(NameError, PatsyError))
# But then again it should, if called from one down on the stack:
def check_nested_call():
x_in_env = "asdf"
t("~ x_in_env", {}, 1, True, [[1, 1], [1, 2], [1, 3]],
["Intercept", "x_in_env"])
check_nested_call()
# passing in an explicit EvalEnvironment also works:
e = EvalEnvironment.capture(1)
t_invalid("~ x_in_env", {}, e, exc=(NameError, PatsyError))
e = EvalEnvironment.capture(0)
def check_nested_call_2():
x_in_env = "asdf"
t("~ x_in_env", {}, e, True, [[1, 1], [1, 2], [1, 3]],
["Intercept", "x_in_env"])
check_nested_call_2()
def build_design_matrices(design_infos, data,
NA_action="drop",
return_type="matrix",
dtype=np.dtype(float)):
"""Construct several design matrices from :class:`DesignMatrixBuilder`
objects.
This is one of Patsy's fundamental functions. This function and
:func:`design_matrix_builders` together form the API to the core formula
interpretation machinery.
:arg design_infos: A list of :class:`DesignInfo` objects describing the
design matrices to be built.
:arg data: A dict-like object which will be used to look up data.
:arg NA_action: What to do with rows that contain missing values. You can
``"drop"`` them, ``"raise"`` an error, or for customization, pass an
:class:`NAAction` object. See :class:`NAAction` for details on what
values count as 'missing' (and how to alter this).
:arg return_type: Either ``"matrix"`` or ``"dataframe"``. See below.
:arg dtype: The dtype of the returned matrix. Useful if you want to use
single-precision or extended-precision.
This function returns either a list of :class:`DesignMatrix` objects (for
``return_type="matrix"``) or a list of :class:`pandas.DataFrame` objects
(for ``return_type="dataframe"``). In both cases, all returned design
matrices will have ``.design_info`` attributes containing the appropriate
:class:`DesignInfo` objects.
Note that unlike :func:`design_matrix_builders`, this function takes only
a simple data argument, not any kind of iterator. That's because this
function doesn't need a global view of the data -- everything that depends
on the whole data set is already encapsulated in the ``design_infos``. If
you are incrementally processing a large data set, simply call this
function for each chunk.
Index handling: This function always checks for indexes in the following
places:
* If ``data`` is a :class:`pandas.DataFrame`, its ``.index`` attribute.
* If any factors evaluate to a :class:`pandas.Series` or
:class:`pandas.DataFrame`, then their ``.index`` attributes.
If multiple indexes are found, they must be identical (same values in the
same order). If no indexes are found, then a default index is generated
using ``np.arange(num_rows)``. One way or another, we end up with a single
index for all the data. If ``return_type="dataframe"``, then this index is
used as the index of the returned DataFrame objects. Examining this index
makes it possible to determine which rows were removed due to NAs.
Determining the number of rows in design matrices: This is not as obvious
as it might seem, because it's possible to have a formula like "~ 1" that
doesn't depend on the data (it has no factors). For this formula, it's
obvious what every row in the design matrix should look like (just the
value ``1``); but, how many rows like this should there be? To determine
the number of rows in a design matrix, this function always checks in the
following places:
* If ``data`` is a :class:`pandas.DataFrame`, then its number of rows.
* The number of entries in any factors present in any of the design
* matrices being built.
All these values much match. In particular, if this function is called to
generate multiple design matrices at once, then they must all have the
same number of rows.
.. versionadded:: 0.2.0
The ``NA_action`` argument.
"""
if isinstance(NA_action, str):
NA_action = NAAction(NA_action)
if return_type == "dataframe" and not have_pandas:
raise PatsyError("pandas.DataFrame was requested, but pandas "
"is not installed")
if return_type not in ("matrix", "dataframe"):
raise PatsyError("unrecognized output type %r, should be "
"'matrix' or 'dataframe'" % (return_type,))
# Evaluate factors
factor_info_to_values = {}
factor_info_to_isNAs = {}
rows_checker = _CheckMatch("Number of rows", lambda a, b: a == b)
index_checker = _CheckMatch("Index", lambda a, b: a.equals(b))
if have_pandas and isinstance(data, pandas.DataFrame):
index_checker.check(data.index, "data.index", None)
rows_checker.check(data.shape[0], "data argument", None)
for design_info in design_infos:
# We look at evaluators rather than factors here, because it might
# happen that we have the same factor twice, but with different
# memorized state.
for factor_info in six.itervalues(design_info.factor_infos):
if factor_info not in factor_info_to_values:
value, is_NA = _eval_factor(factor_info, data, NA_action)
factor_info_to_isNAs[factor_info] = is_NA
# value may now be a Series, DataFrame, or ndarray
name = factor_info.factor.name()
origin = factor_info.factor.origin
rows_checker.check(value.shape[0], name, origin)
if (have_pandas
and isinstance(value, (pandas.Series, pandas.DataFrame))):
index_checker.check(value.index, name, origin)
# Strategy: we work with raw ndarrays for doing the actual
# combining; DesignMatrixBuilder objects never sees pandas
# objects. Then at the end, if a DataFrame was requested, we
# convert. So every entry in this dict is either a 2-d array
# of floats, or a 1-d array of integers (representing
# categories).
value = np.asarray(value)
factor_info_to_values[factor_info] = value
# Handle NAs
values = list(factor_info_to_values.values())
is_NAs = list(factor_info_to_isNAs.values())
origins = [factor_info.factor.origin
for factor_info in factor_info_to_values]
pandas_index = index_checker.value
num_rows = rows_checker.value
# num_rows is None iff evaluator_to_values (and associated sets like
# 'values') are empty, i.e., we have no actual evaluators involved
# (formulas like "~ 1").
if return_type == "dataframe" and num_rows is not None:
if pandas_index is None:
pandas_index = np.arange(num_rows)
values.append(pandas_index)
is_NAs.append(np.zeros(len(pandas_index), dtype=bool))
origins.append(None)
new_values = NA_action.handle_NA(values, is_NAs, origins)
# NA_action may have changed the number of rows.
if new_values:
num_rows = new_values[0].shape[0]
if return_type == "dataframe" and num_rows is not None:
pandas_index = new_values.pop()
factor_info_to_values = dict(zip(factor_info_to_values, new_values))
# Build factor values into matrices
results = []
for design_info in design_infos:
results.append(_build_design_matrix(design_info,
factor_info_to_values,
dtype))
matrices = []
for need_reshape, matrix in results:
if need_reshape:
# There is no data-dependence, at all -- a formula like "1 ~ 1".
# In this case the builder just returns a single-row matrix, and
# we have to broadcast it vertically to the appropriate size. If
# we can figure out what that is...
assert matrix.shape[0] == 1
if num_rows is not None:
matrix = DesignMatrix(np.repeat(matrix, num_rows, axis=0),
matrix.design_info)
else:
raise PatsyError(
"No design matrix has any non-trivial factors, "
"the data object is not a DataFrame. "
"I can't tell how many rows the design matrix should "
"have!"
)
matrices.append(matrix)
if return_type == "dataframe":
assert have_pandas
for i, matrix in enumerate(matrices):
di = matrix.design_info
matrices[i] = pandas.DataFrame(matrix,
columns=di.column_names,
index=pandas_index)
matrices[i].design_info = di
return matrices
def build_design_matrices(builders, data,
NA_action="drop",
return_type="matrix",
dtype=np.dtype(float)):
"""Construct several design matrices from :class:`DesignMatrixBuilder`
objects.
This is one of Patsy's fundamental functions. This function and
:func:`design_matrix_builders` together form the API to the core formula
interpretation machinery.
:arg builders: A list of :class:`DesignMatrixBuilders` specifying the
design matrices to be built.
:arg data: A dict-like object which will be used to look up data.
:arg NA_action: What to do with rows that contain missing values. You can
``"drop"`` them, ``"raise"`` an error, or for customization, pass an
:class:`NAAction` object. See :class:`NAAction` for details on what
values count as 'missing' (and how to alter this).
:arg return_type: Either ``"matrix"`` or ``"dataframe"``. See below.
:arg dtype: The dtype of the returned matrix. Useful if you want to use
single-precision or extended-precision.
This function returns either a list of :class:`DesignMatrix` objects (for
``return_type="matrix"``) or a list of :class:`pandas.DataFrame` objects
(for ``return_type="dataframe"``). In the latter case, the DataFrames will
preserve any (row) indexes that were present in the input, which may be
useful for time-series models etc. In any case, all returned design
matrices will have ``.design_info`` attributes containing the appropriate
:class:`DesignInfo` objects.
Unlike :func:`design_matrix_builders`, this function takes only a simple
data argument, not any kind of iterator. That's because this function
doesn't need a global view of the data -- everything that depends on the
whole data set is already encapsulated in the `builders`. If you are
incrementally processing a large data set, simply call this function for
each chunk.
"""
if isinstance(NA_action, basestring):
NA_action = NAAction(NA_action)
if return_type == "dataframe" and not have_pandas:
raise PatsyError("pandas.DataFrame was requested, but pandas "
"is not installed")
if return_type not in ("matrix", "dataframe"):
raise PatsyError("unrecognized output type %r, should be "
"'matrix' or 'dataframe'" % (return_type,))
# Evaluate factors
evaluator_to_values = {}
evaluator_to_isNAs = {}
num_rows = None
pandas_index = None
for builder in builders:
# We look at evaluators rather than factors here, because it might
# happen that we have the same factor twice, but with different
# memorized state.
for evaluator in builder._evaluators:
if evaluator not in evaluator_to_values:
value, is_NA = evaluator.eval(data, NA_action)
evaluator_to_isNAs[evaluator] = is_NA
# value may now be a Series, DataFrame, or ndarray
if num_rows is None:
num_rows = value.shape[0]
else:
if num_rows != value.shape[0]:
msg = ("Row mismatch: factor %s had %s rows, when "
"previous factors had %s rows"
% (evaluator.factor.name(), value.shape[0],
num_rows))
raise PatsyError(msg, evaluator.factor)
if (have_pandas
and isinstance(value, (pandas.Series, pandas.DataFrame))):
if pandas_index is None:
pandas_index = value.index
else:
if not pandas_index.equals(value.index):
msg = ("Index mismatch: pandas objects must "
"have aligned indexes")
raise PatsyError(msg, evaluator.factor)
# Strategy: we work with raw ndarrays for doing the actual
# combining; DesignMatrixBuilder objects never sees pandas
# objects. Then at the end, if a DataFrame was requested, we
# convert. So every entry in this dict is either a 2-d array
# of floats, or a 1-d array of integers (representing
# categories).
value = np.asarray(value)
evaluator_to_values[evaluator] = value
# Handle NAs
values = evaluator_to_values.values()
is_NAs = evaluator_to_isNAs.values()
if return_type == "dataframe" and num_rows is not None:
if pandas_index is None:
pandas_index = np.arange(num_rows)
values.append(pandas_index)
is_NAs.append(np.zeros(len(pandas_index), dtype=bool))
origins = [evaluator.factor.origin for evaluator in evaluator_to_values]
new_values = NA_action.handle_NA(values, is_NAs, origins)
if return_type == "dataframe" and num_rows is not None:
pandas_index = new_values.pop()
evaluator_to_values = dict(zip(evaluator_to_values, new_values))
# Build factor values into matrices
results = []
for builder in builders:
results.append(builder._build(evaluator_to_values, dtype))
matrices = []
for need_reshape, matrix in results:
if need_reshape and num_rows is not None:
assert matrix.shape[0] == 1
matrices.append(DesignMatrix(np.repeat(matrix, num_rows, axis=0),
matrix.design_info))
else:
# There is no data-dependence, at all -- a formula like "1 ~ 1". I
# guess we'll just return some single-row matrices. Perhaps it
# would be better to figure out how many rows are in the input
# data and broadcast to that size, but eh. Input data is optional
# in the first place, so even that would be no guarantee... let's
# wait until someone actually has a relevant use case before we
# worry about it.
matrices.append(matrix)
if return_type == "dataframe":
assert have_pandas
for i, matrix in enumerate(matrices):
di = matrix.design_info
matrices[i] = pandas.DataFrame(matrix,
columns=di.column_names,
index=pandas_index)
matrices[i].design_info = di
return matrices