def test_cfm_approximate_correct_ordering(): trips = { 'trip_id': [i for i in range(1000)], 'flight_time': [datetime(1998, 4, 2) for i in range(350)] + [datetime(1997, 4, 3) for i in range(650)], 'flight_id': [randint(1, 25) for i in range(1000)], 'trip_duration': [randint(1, 999) for i in range(1000)] } df = pd.DataFrame.from_dict(trips) es = EntitySet('flights') es.entity_from_dataframe("trips", dataframe=df, index="trip_id", time_index='flight_time') es.normalize_entity(base_entity_id="trips", new_entity_id="flights", index="flight_id", make_time_index=True) features = dfs(entityset=es, target_entity='trips', features_only=True) flight_features = [ feature for feature in features if isinstance(feature, DirectFeature) and isinstance(feature.base_features[0], AggregationPrimitive) ] property_feature = IdentityFeature(es['trips']['trip_id']) # direct_agg_feat = DirectFeature(Sum(es['trips']['trip_duration'], # es['flights']), # es['trips']) cutoff_time = pd.DataFrame.from_dict({ 'instance_id': df['trip_id'], 'time': df['flight_time'] }) time_feature = IdentityFeature(es['trips']['flight_time']) feature_matrix = calculate_feature_matrix(flight_features + [property_feature, time_feature], es, cutoff_time_in_index=True, cutoff_time=cutoff_time) feature_matrix.index.names = ['instance', 'time'] assert (np.all( feature_matrix.reset_index('time').reset_index()[['instance', 'time']]. values == feature_matrix[['trip_id', 'flight_time']].values)) feature_matrix_2 = calculate_feature_matrix( flight_features + [property_feature, time_feature], es, cutoff_time=cutoff_time, cutoff_time_in_index=True, approximate=Timedelta(2, 'd')) feature_matrix_2.index.names = ['instance', 'time'] assert (np.all( feature_matrix_2.reset_index('time').reset_index()[[ 'instance', 'time' ]].values == feature_matrix_2[['trip_id', 'flight_time']].values)) for column in feature_matrix: for x, y in zip(feature_matrix[column], feature_matrix_2[column]): if not ((pd.isnull(x) and pd.isnull(y)) or (x == y)): import pdb pdb.set_trace() assert ((pd.isnull(x) and pd.isnull(y)) or (x == y))
def _make_entity_set(self, X): """Helper method that creates and returns the entity set given the input data""" ft_es = EntitySet() if self.index not in X.columns: es = ft_es.entity_from_dataframe(entity_id="X", dataframe=X, index=self.index, make_index=True) else: es = ft_es.entity_from_dataframe(entity_id="X", dataframe=X, index=self.index) return es
def test_encode_unknown_features(): # Dataframe with categorical column with "unknown" string df = pd.DataFrame({'category': ['unknown', 'b', 'c', 'd', 'e']}) es = EntitySet('test') es.entity_from_dataframe(entity_id='a', dataframe=df, index='index', make_index=True) features, feature_defs = dfs(entityset=es, target_entity='a') # Specify unknown token for replacement features_enc, feature_defs_enc = encode_features(features, feature_defs, include_unknown=True) assert list(features_enc.columns) == ['category = unknown', 'category = e', 'category = d', 'category = c', 'category = b', 'category is unknown']
def test_encode_features_drop_first(): df = pd.DataFrame({'category': ['ao', 'b', 'c', 'd', 'e']}) es = EntitySet('test') es.entity_from_dataframe(entity_id='a', dataframe=df, index='index', make_index=True) features, feature_defs = dfs(entityset=es, target_entity='a') features_enc, feature_defs_enc = encode_features(features, feature_defs, drop_first=True, include_unknown=False) assert len(features_enc.columns) == 4 features_enc, feature_defs = encode_features(features, feature_defs, top_n=3, drop_first=True, include_unknown=False) assert len(features_enc.columns) == 2
def test_encode_unknown_features(): # Dataframe with categorical column with "unknown" string df = pd.DataFrame({'category': ['unknown', 'b', 'c', 'd', 'e']}) es = EntitySet('test') es.entity_from_dataframe(entity_id='a', dataframe=df, index='index', make_index=True) features, feature_defs = dfs(entityset=es, target_entity='a') # Specify unknown token for replacement features_enc, feature_defs_enc = encode_features(features, feature_defs, include_unknown=True) assert list(features_enc.columns) == ['category = unknown', 'category = e', 'category = d', 'category = c', 'category = b', 'category is unknown']
def test_cfm_approximate_correct_ordering(): trips = { 'trip_id': [i for i in range(1000)], 'flight_time': [datetime(1998, 4, 2) for i in range(350)] + [datetime(1997, 4, 3) for i in range(650)], 'flight_id': [randint(1, 25) for i in range(1000)], 'trip_duration': [randint(1, 999) for i in range(1000)] } df = pd.DataFrame.from_dict(trips) es = EntitySet('flights') es.entity_from_dataframe("trips", dataframe=df, index="trip_id", time_index='flight_time') es.normalize_entity(base_entity_id="trips", new_entity_id="flights", index="flight_id", make_time_index=True) features = dfs(entityset=es, target_entity='trips', features_only=True) flight_features = [feature for feature in features if isinstance(feature, DirectFeature) and isinstance(feature.base_features[0], AggregationPrimitive)] property_feature = IdentityFeature(es['trips']['trip_id']) # direct_agg_feat = DirectFeature(Sum(es['trips']['trip_duration'], # es['flights']), # es['trips']) cutoff_time = pd.DataFrame.from_dict({'instance_id': df['trip_id'], 'time': df['flight_time']}) time_feature = IdentityFeature(es['trips']['flight_time']) feature_matrix = calculate_feature_matrix(flight_features + [property_feature, time_feature], cutoff_time_in_index=True, cutoff_time=cutoff_time) feature_matrix.index.names = ['instance', 'time'] assert(np.all(feature_matrix.reset_index('time').reset_index()[['instance', 'time']].values == feature_matrix[['trip_id', 'flight_time']].values)) feature_matrix_2 = calculate_feature_matrix(flight_features + [property_feature, time_feature], cutoff_time=cutoff_time, cutoff_time_in_index=True, approximate=Timedelta(2, 'd')) feature_matrix_2.index.names = ['instance', 'time'] assert(np.all(feature_matrix_2.reset_index('time').reset_index()[['instance', 'time']].values == feature_matrix_2[['trip_id', 'flight_time']].values)) for column in feature_matrix: for x, y in zip(feature_matrix[column], feature_matrix_2[column]): if not ((pd.isnull(x) and pd.isnull(y)) or (x == y)): import pdb pdb.set_trace() assert ((pd.isnull(x) and pd.isnull(y)) or (x == y))
def test_encode_features_matches_calculate_feature_matrix(): df = pd.DataFrame({'category': ['b', 'c', 'd', 'e']}) pd_es = EntitySet('test') pd_es.entity_from_dataframe(entity_id='a', dataframe=df, index='index', make_index=True) features, feature_defs = dfs(entityset=pd_es, target_entity='a') features_enc, feature_defs_enc = encode_features(features, feature_defs, to_encode=['category']) features_calc = calculate_feature_matrix(feature_defs_enc, entityset=pd_es) assert features_enc['category = e'].dtypes == bool assert features_enc['category = e'].dtypes == features_calc[ 'category = e'].dtypes
def main(): holding = Holding( 'HDFCBANK', Price('buy', 97.89), 5, 112.1, 95.4 ) print(holding.to_json()) es = EntitySet('user').entity_from_dataframe( entity_id='parent', dataframe=pd.DataFrame({ 'id': [10, 14, 24, 34, 54, 64, 84, ], 'age': [10, 14, 24, 34, 54, 64, 84, ], 'salary': [1, 1, 10000, 25000, 50000, 60000, 80000, ] })) es.entity_from_dataframe( entity_id='child', dataframe=pd.DataFrame({ 'id': [10, 14, 24, 34, 54, 64, 84, ], 'user_id': [10, 14, 24, 34, 54, 64, 84, ], 'sex': [0, 1, 0, 1, 1, 0, 1] })) es.add_relationship( Relationship( es['parent']['id'], es['child']['user_id'] ) ) es.entity_from_dataframe( entity_id='child_0', dataframe=pd.DataFrame({ 'id': [10, 14, 24, 34, 54, 64, 84, ], 'user_id': [10, 14, 24, 34, 54, 64, 84, ], 'diet': [0, 1, 0, 1, 1, 0, 1] })) es.add_relationship( Relationship( es['parent']['id'], es['child_0']['user_id'] ) ) # print(es) e_s = CustomEntity(entity_set=es) print(e_s.to_json())