def test_groupby_transform(es):
feat = GroupByTransformFeature(es['customers']['age'], CumMax,
es['customers']['cohort'])
graph = graph_feature(feat).source
feat_name = feat.get_name()
prim_node = "0_{}_cum_max".format(feat_name)
groupby_node = '{}_groupby_customers--cohort'.format(feat_name)
entity_table = '\u2605 customers (target)'
groupby_edge = 'customers:cohort -> "{}"'.format(groupby_node)
groupby_input = 'customers:age -> "{}"'.format(groupby_node)
prim_input = '"{}" -> "{}"'.format(groupby_node, prim_node)
feat_edge = '"{}" -> customers:"{}"'.format(prim_node, feat_name)
graph_components = [
feat_name, prim_node, groupby_node, entity_table, groupby_edge,
groupby_input, prim_input, feat_edge
]
for component in graph_components:
assert component in graph
matches = re.findall(r"customers \[label=<\n<TABLE.*?</TABLE>>", graph,
re.DOTALL)
assert len(matches) == 1
rows = re.findall(r"<TR.*?</TR>", matches[0], re.DOTALL)
assert len(rows) == 4
assert entity_table in rows[0]
assert feat_name in rows[-1]
assert ('age' in rows[1] and 'cohort' in rows[2]) or \
('age' in rows[2] and 'cohort' in rows[1])
def test_groupby_transform_direct_groupby(es):
groupby = DirectFeature(IdentityFeature(es['cohorts'].ww['cohort_name']),
'customers')
feat = GroupByTransformFeature(IdentityFeature(es['customers'].ww['age']),
CumMax, groupby)
graph = graph_feature(feat).source
groupby_name = groupby.get_name()
feat_name = feat.get_name()
join_node = '1_{}_join'.format(groupby_name)
prim_node = "0_{}_cum_max".format(feat_name)
groupby_node = '{}_groupby_customers--{}'.format(feat_name, groupby_name)
customers_table = '\u2605 customers (target)'
cohorts_table = 'cohorts'
join_groupby = '"{}" -> customers:cohort'.format(join_node)
join_input = 'cohorts:cohort_name -> "{}"'.format(join_node)
join_out_edge = '"{}" -> customers:"{}"'.format(join_node, groupby_name)
groupby_edge = 'customers:"{}" -> "{}"'.format(groupby_name, groupby_node)
groupby_input = 'customers:age -> "{}"'.format(groupby_node)
prim_input = '"{}" -> "{}"'.format(groupby_node, prim_node)
feat_edge = '"{}" -> customers:"{}"'.format(prim_node, feat_name)
graph_components = [
groupby_name, feat_name, join_node, prim_node, groupby_node,
customers_table, cohorts_table, join_groupby, join_input,
join_out_edge, groupby_edge, groupby_input, prim_input, feat_edge
]
for component in graph_components:
assert component in graph
dataframes = {
'cohorts': [cohorts_table, 'cohort_name'],
'customers':
[customers_table, 'cohort', 'age', groupby_name, feat_name]
}
for dataframe in dataframes:
regex = r"{} \[label=<\n<TABLE.*?</TABLE>>".format(dataframe)
matches = re.findall(regex, graph, re.DOTALL)
assert len(matches) == 1
rows = re.findall(r"<TR.*?</TR>", matches[0], re.DOTALL)
assert len(rows) == len(dataframes[dataframe])
for row in rows:
matched = False
for i in dataframes[dataframe]:
if i in row:
matched = True
dataframes[dataframe].remove(i)
break
assert matched
def test_metadata(es, tmpdir):
identity_feature_descriptions = {'sessions: device_name': 'the name of the device used for each session',
'customers: id': "the customer's id"}
agg_feat = AggregationFeature(IdentityFeature(es['sessions'].ww['device_name']), 'customers', NumUnique)
agg_description = 'The number of unique elements in the name of the device used for each '\
'session of all instances of "sessions" for each customer\'s id.'
assert describe_feature(agg_feat, feature_descriptions=identity_feature_descriptions) == agg_description
transform_feat = GroupByTransformFeature(IdentityFeature(es['log'].ww['value']), CumMean, IdentityFeature(es['log'].ww['session_id']))
transform_description = 'The running average of the "value" for each "session_id".'
primitive_templates = {"cum_mean": "the running average of {}"}
assert describe_feature(transform_feat, primitive_templates=primitive_templates) == transform_description
custom_agg = AggregationFeature(IdentityFeature(es['log'].ww['zipcode']), 'sessions', Mode)
auto_description = 'The most frequently occurring value of the "zipcode" of all instances of "log" for each "id" in "sessions".'
custom_agg_description = "the most frequently used zipcode"
custom_feature_description = custom_agg_description[0].upper() + custom_agg_description[1:] + '.'
feature_description_dict = {'sessions: MODE(log.zipcode)': custom_agg_description}
assert describe_feature(custom_agg) == auto_description
assert describe_feature(custom_agg, feature_descriptions=feature_description_dict) == custom_feature_description
metadata = {
'feature_descriptions': {**identity_feature_descriptions, **feature_description_dict},
'primitive_templates': primitive_templates
}
metadata_path = os.path.join(tmpdir, 'description_metadata.json')
with open(metadata_path, 'w') as f:
json.dump(metadata, f)
assert describe_feature(agg_feat, metadata_file=metadata_path) == agg_description
assert describe_feature(transform_feat, metadata_file=metadata_path) == transform_description
assert describe_feature(custom_agg, metadata_file=metadata_path) == custom_feature_description
def test_groupby_transform_description(es):
feature = GroupByTransformFeature(
IdentityFeature(es["log"].ww["value"]),
CumMean,
IdentityFeature(es["log"].ww["session_id"]),
)
description = 'The cumulative mean of the "value" for each "session_id".'
assert describe_feature(feature) == description
def test_groupby_transform(es):
feat = GroupByTransformFeature(
IdentityFeature(es["customers"].ww["age"]),
CumMax,
IdentityFeature(es["customers"].ww["cohort"]),
)
graph = graph_feature(feat).source
feat_name = feat.get_name()
prim_node = "0_{}_cum_max".format(feat_name)
groupby_node = "{}_groupby_customers--cohort".format(feat_name)
dataframe_table = "\u2605 customers (target)"
groupby_edge = 'customers:cohort -> "{}"'.format(groupby_node)
groupby_input = 'customers:age -> "{}"'.format(groupby_node)
prim_input = '"{}" -> "{}"'.format(groupby_node, prim_node)
feat_edge = '"{}" -> customers:"{}"'.format(prim_node, feat_name)
graph_components = [
feat_name,
prim_node,
groupby_node,
dataframe_table,
groupby_edge,
groupby_input,
prim_input,
feat_edge,
]
for component in graph_components:
assert component in graph
matches = re.findall(r"customers \[label=<\n<TABLE.*?</TABLE>>", graph, re.DOTALL)
assert len(matches) == 1
rows = re.findall(r"<TR.*?</TR>", matches[0], re.DOTALL)
assert len(rows) == 4
assert dataframe_table in rows[0]
assert feat_name in rows[-1]
assert ("age" in rows[1] and "cohort" in rows[2]) or (
"age" in rows[2] and "cohort" in rows[1]
)
def _build_transform_features(self,
all_features,
entity,
max_depth=0,
require_direct_input=False):
"""Creates trans_features for all the variables in an entity
Args:
all_features (dict[:class:`.Entity`.id:dict->[str->:class:`BaseFeature`]]):
Dict containing a dict for each entity. Each nested dict
has features as values with their ids as keys
entity (Entity): Entity to calculate features for.
"""
new_max_depth = None
if max_depth is not None:
new_max_depth = max_depth - 1
for trans_prim in self.trans_primitives:
current_options = self.primitive_options[trans_prim.name]
if ignore_entity_for_primitive(current_options, entity):
continue
# if multiple input_types, only use first one for DFS
input_types = trans_prim.input_types
if type(input_types[0]) == list:
input_types = input_types[0]
matching_inputs = self._get_matching_inputs(
all_features,
entity,
new_max_depth,
input_types,
trans_prim,
current_options,
require_direct_input=require_direct_input)
for matching_input in matching_inputs:
if all(bf.number_output_features == 1
for bf in matching_input):
new_f = TransformFeature(matching_input,
primitive=trans_prim)
self._handle_new_feature(all_features=all_features,
new_feature=new_f)
for groupby_prim in self.groupby_trans_primitives:
current_options = self.primitive_options[groupby_prim.name]
if ignore_entity_for_primitive(current_options,
entity,
groupby=True):
continue
input_types = groupby_prim.input_types[:]
# if multiple input_types, only use first one for DFS
if type(input_types[0]) == list:
input_types = input_types[0]
matching_inputs = self._get_matching_inputs(
all_features, entity, new_max_depth, input_types, groupby_prim,
current_options)
# get columns to use as groupbys, use IDs as default unless other groupbys specified
if any([
'include_groupby_variables' in option
and entity.id in option['include_groupby_variables']
for option in current_options
]):
default_type = variable_types.PandasTypes._all
else:
default_type = set([Id])
groupby_matches = self._features_by_type(
all_features=all_features,
entity=entity,
max_depth=new_max_depth,
variable_type=default_type)
groupby_matches = filter_groupby_matches_by_options(
groupby_matches, current_options)
# If require_direct_input, require a DirectFeature in input or as a
# groupby, and don't create features of inputs/groupbys which are
# all direct features with the same relationship path
for matching_input in matching_inputs:
if all(bf.number_output_features == 1
for bf in matching_input):
for groupby in groupby_matches:
if require_direct_input and (
_all_direct_and_same_path(matching_input +
(groupby, ))
or not any([
isinstance(feature, DirectFeature)
for feature in (matching_input +
(groupby, ))
])):
continue
new_f = GroupByTransformFeature(list(matching_input),
groupby=groupby[0],
primitive=groupby_prim)
self._handle_new_feature(all_features=all_features,
new_feature=new_f)
def test_groupby_transform_description(es):
feature = GroupByTransformFeature(es['log']['value'], CumMean,
es['log']['session_id'])
description = 'The cumulative mean of the "value" for each "session_id".'
assert describe_feature(feature) == description
def test_metadata(es, tmpdir):
identity_feature_descriptions = {
"sessions: device_name": "the name of the device used for each session",
"customers: id": "the customer's id",
}
agg_feat = AggregationFeature(
IdentityFeature(es["sessions"].ww["device_name"]), "customers", NumUnique
)
agg_description = (
"The number of unique elements in the name of the device used for each "
'session of all instances of "sessions" for each customer\'s id.'
)
assert (
describe_feature(agg_feat, feature_descriptions=identity_feature_descriptions)
== agg_description
)
transform_feat = GroupByTransformFeature(
IdentityFeature(es["log"].ww["value"]),
CumMean,
IdentityFeature(es["log"].ww["session_id"]),
)
transform_description = 'The running average of the "value" for each "session_id".'
primitive_templates = {"cum_mean": "the running average of {}"}
assert (
describe_feature(transform_feat, primitive_templates=primitive_templates)
== transform_description
)
custom_agg = AggregationFeature(
IdentityFeature(es["log"].ww["zipcode"]), "sessions", Mode
)
auto_description = 'The most frequently occurring value of the "zipcode" of all instances of "log" for each "id" in "sessions".'
custom_agg_description = "the most frequently used zipcode"
custom_feature_description = (
custom_agg_description[0].upper() + custom_agg_description[1:] + "."
)
feature_description_dict = {"sessions: MODE(log.zipcode)": custom_agg_description}
assert describe_feature(custom_agg) == auto_description
assert (
describe_feature(custom_agg, feature_descriptions=feature_description_dict)
== custom_feature_description
)
metadata = {
"feature_descriptions": {
**identity_feature_descriptions,
**feature_description_dict,
},
"primitive_templates": primitive_templates,
}
metadata_path = os.path.join(tmpdir, "description_metadata.json")
with open(metadata_path, "w") as f:
json.dump(metadata, f)
assert describe_feature(agg_feat, metadata_file=metadata_path) == agg_description
assert (
describe_feature(transform_feat, metadata_file=metadata_path)
== transform_description
)
assert (
describe_feature(custom_agg, metadata_file=metadata_path)
== custom_feature_description
)
def _build_transform_features(self, all_features, entity, max_depth=0):
"""Creates trans_features for all the variables in an entity
Args:
all_features (dict[:class:`.Entity`.id:dict->[str->:class:`BaseFeature`]]):
Dict containing a dict for each entity. Each nested dict
has features as values with their ids as keys
entity (Entity): Entity to calculate features for.
"""
if max_depth is not None and max_depth < 0:
return
new_max_depth = None
if max_depth is not None:
new_max_depth = max_depth - 1
self._add_identity_features(all_features, entity)
for trans_prim in self.trans_primitives:
# if multiple input_types, only use first one for DFS
input_types = trans_prim.input_types
if type(input_types[0]) == list:
input_types = input_types[0]
features = self._features_by_type(all_features=all_features,
entity=entity,
max_depth=new_max_depth,
variable_type=set(input_types))
matching_inputs = match(input_types,
features,
commutative=trans_prim.commutative)
for matching_input in matching_inputs:
if all(bf.number_output_features == 1
for bf in matching_input):
new_f = TransformFeature(matching_input,
primitive=trans_prim)
self._handle_new_feature(all_features=all_features,
new_feature=new_f)
for groupby_prim in self.groupby_trans_primitives:
# Normally input_types is a list of what inputs can be supplied to
# the primitive function. Here we temporarily add `Id` as an extra
# item in input_types so that the matching function will also look
# for feature columns to group by.
input_types = groupby_prim.input_types[:]
# if multiple input_types, only use first one for DFS
if type(input_types[0]) == list:
input_types = input_types[0]
input_types.append(Id)
features = self._features_by_type(all_features=all_features,
entity=entity,
max_depth=new_max_depth,
variable_type=set(input_types))
matching_inputs = match(input_types,
features,
commutative=groupby_prim.commutative)
for matching_input in matching_inputs:
if all(bf.number_output_features == 1
for bf in matching_input):
new_f = GroupByTransformFeature(list(matching_input[:-1]),
groupby=matching_input[-1],
primitive=groupby_prim)
self._handle_new_feature(all_features=all_features,
new_feature=new_f)
def _build_transform_features(self,
all_features,
dataframe,
max_depth=0,
require_direct_input=False):
"""Creates trans_features for all the columns in a dataframe
Args:
all_features (dict[dataframe name: dict->[str->:class:`BaseFeature`]]):
Dict containing a dict for each dataframe. Each nested dict
has features as values with their ids as keys
dataframe (DataFrame): DataFrame to calculate features for.
"""
new_max_depth = None
if max_depth is not None:
new_max_depth = max_depth - 1
# Keep track of features to add until the end to avoid applying
# transform primitives to features that were also built by transform primitives
features_to_add = []
for trans_prim in self.trans_primitives:
current_options = self.primitive_options.get(
trans_prim, self.primitive_options.get(trans_prim.name))
if ignore_dataframe_for_primitive(current_options, dataframe):
continue
input_types = trans_prim.input_types
matching_inputs = self._get_matching_inputs(
all_features,
dataframe,
new_max_depth,
input_types,
trans_prim,
current_options,
require_direct_input=require_direct_input,
)
for matching_input in matching_inputs:
if all(bf.number_output_features == 1
for bf in matching_input) and check_transform_stacking(
matching_input):
new_f = TransformFeature(matching_input,
primitive=trans_prim)
features_to_add.append(new_f)
for groupby_prim in self.groupby_trans_primitives:
current_options = self.primitive_options.get(
groupby_prim, self.primitive_options.get(groupby_prim.name))
if ignore_dataframe_for_primitive(current_options,
dataframe,
groupby=True):
continue
input_types = groupby_prim.input_types[:]
matching_inputs = self._get_matching_inputs(
all_features,
dataframe,
new_max_depth,
input_types,
groupby_prim,
current_options,
)
# get columns to use as groupbys, use IDs as default unless other groupbys specified
if any([
"include_groupby_columns" in option
and dataframe.ww.name in option["include_groupby_columns"]
for option in current_options
]):
column_schemas = "all"
else:
column_schemas = [ColumnSchema(semantic_tags=["foreign_key"])]
groupby_matches = self._features_by_type(
all_features=all_features,
dataframe=dataframe,
max_depth=new_max_depth,
column_schemas=column_schemas,
)
groupby_matches = filter_groupby_matches_by_options(
groupby_matches, current_options)
# If require_direct_input, require a DirectFeature in input or as a
# groupby, and don't create features of inputs/groupbys which are
# all direct features with the same relationship path
for matching_input in matching_inputs:
if all(bf.number_output_features == 1
for bf in matching_input) and check_transform_stacking(
matching_input):
for groupby in groupby_matches:
if require_direct_input and (
_all_direct_and_same_path(matching_input +
(groupby, ))
or not any([
isinstance(feature, DirectFeature)
for feature in (matching_input +
(groupby, ))
])):
continue
new_f = GroupByTransformFeature(
list(matching_input),
groupby=groupby[0],
primitive=groupby_prim,
)
features_to_add.append(new_f)
for new_f in features_to_add:
self._handle_new_feature(all_features=all_features,
new_feature=new_f)
def _build_transform_features(self,
all_features,
entity,
max_depth=0,
require_direct_input=False):
"""Creates trans_features for all the variables in an entity
Args:
all_features (dict[:class:`.Entity`.id:dict->[str->:class:`BaseFeature`]]):
Dict containing a dict for each entity. Each nested dict
has features as values with their ids as keys
entity (Entity): Entity to calculate features for.
"""
new_max_depth = None
if max_depth is not None:
new_max_depth = max_depth - 1
for trans_prim in self.trans_primitives:
# if multiple input_types, only use first one for DFS
input_types = trans_prim.input_types
if type(input_types[0]) == list:
input_types = input_types[0]
matching_inputs = self._get_matching_inputs(
all_features,
entity,
new_max_depth,
input_types,
trans_prim,
require_direct_input=require_direct_input)
for matching_input in matching_inputs:
if all(bf.number_output_features == 1
for bf in matching_input):
new_f = TransformFeature(matching_input,
primitive=trans_prim)
self._handle_new_feature(all_features=all_features,
new_feature=new_f)
for groupby_prim in self.groupby_trans_primitives:
input_types = groupby_prim.input_types[:]
# if multiple input_types, only use first one for DFS
if type(input_types[0]) == list:
input_types = input_types[0]
matching_inputs = self._get_matching_inputs(
all_features,
entity,
new_max_depth,
input_types,
groupby_prim,
require_direct_input=require_direct_input)
# get IDs to use as groupby
id_matches = self._features_by_type(all_features=all_features,
entity=entity,
max_depth=new_max_depth,
variable_type=set([Id]))
for matching_input in matching_inputs:
if all(bf.number_output_features == 1
for bf in matching_input):
for id_groupby in id_matches:
new_f = GroupByTransformFeature(list(matching_input),
groupby=id_groupby,
primitive=groupby_prim)
self._handle_new_feature(all_features=all_features,
new_feature=new_f)