def test_fewer_distinct_points_than_cluster(self): input = self.t_env.from_data_stream( self.env.from_collection( [ (Vectors.dense([0.0, 0.1]), ), (Vectors.dense([0.0, 0.1]), ), (Vectors.dense([0.0, 0.1]), ), ], type_info=Types.ROW_NAMED(['features'], [DenseVectorTypeInfo()]))) kmeans = KMeans().set_k(2) model = kmeans.fit(input) output = model.transform(input)[0] results = [ result for result in self.t_env.to_data_stream( output).execute_and_collect() ] field_names = output.get_schema().get_field_names() actual_groups = group_features_by_prediction( results, field_names.index(kmeans.features_col), field_names.index(kmeans.prediction_col)) expected_groups = [{DenseVector([0.0, 0.1])}] self.assertEqual(actual_groups, expected_groups)
def setUp(self): super(OneHotEncoderTest, self).setUp() self.train_data = self.t_env.from_data_stream( self.env.from_collection([ (0.0,), (1.0,), (2.0,), (0.0,), ], type_info=Types.ROW_NAMED( ['input'], [Types.DOUBLE()]))) self.predict_data = self.t_env.from_data_stream( self.env.from_collection([ (0.0,), (1.0,), (2.0,), ], type_info=Types.ROW_NAMED( ['input'], [Types.DOUBLE()]))) self.expected_data = { 0.0: Vectors.sparse(2, [0], [1.0]), 1.0: Vectors.sparse(2, [1], [1.0]), 2.0: Vectors.sparse(2, [], []) } self.estimator = OneHotEncoder().set_input_cols('input').set_output_cols('output')
def test_drop_last(self): self.estimator.set_drop_last(False) expected_data = { 0.0: Vectors.sparse(3, [0], [1.0]), 1.0: Vectors.sparse(3, [1], [1.0]), 2.0: Vectors.sparse(3, [2], [1.0]) } model = self.estimator.fit(self.train_data) # type: OneHotEncoderModel output_table = model.transform(self.predict_data)[0] self.verify_output_result( output_table, model.input_cols, model.output_cols, output_table.get_schema().get_field_names(), expected_data)
def setUp(self): super(LogisticRegressionTest, self).setUp() self.binomial_data_table = self.t_env.from_data_stream( self.env.from_collection( [ (Vectors.dense([1, 2, 3, 4]), 0., 1.), (Vectors.dense([2, 2, 3, 4]), 0., 2.), (Vectors.dense([3, 2, 3, 4]), 0., 3.), (Vectors.dense([4, 2, 3, 4]), 0., 4.), (Vectors.dense([5, 2, 3, 4]), 0., 5.), (Vectors.dense([11, 2, 3, 4]), 1., 1.), (Vectors.dense([12, 2, 3, 4]), 1., 2.), (Vectors.dense([13, 2, 3, 4]), 1., 3.), (Vectors.dense([14, 2, 3, 4]), 1., 4.), (Vectors.dense([15, 2, 3, 4]), 1., 5.), ], type_info=Types.ROW_NAMED( ['features', 'label', 'weight'], [DenseVectorTypeInfo(), Types.DOUBLE(), Types.DOUBLE()])))
def test_max_value_equas_min_value_but_predict_value_not_equals(self): train_data = self.t_env.from_data_stream( self.env.from_collection([ (Vectors.dense([40.0, 80.0]), ), ], type_info=Types.ROW_NAMED( ['input'], [DenseVectorTypeInfo()]))) predict_data = self.t_env.from_data_stream( self.env.from_collection([ (Vectors.dense([30.0, 50.0]), ), ], type_info=Types.ROW_NAMED( ['input'], [DenseVectorTypeInfo()]))) min_max_scalar = MinMaxScaler() \ .set_min(0.0) \ .set_max(10.0) model = min_max_scalar.fit(train_data) result = model.transform(predict_data)[0] self.verify_output_result(result, min_max_scalar.get_output_col(), result.get_schema().get_field_names(), [Vectors.dense(5.0, 5.0)])
def setUp(self): super(VectorAssemblerTest, self).setUp() self.input_data_table = self.t_env.from_data_stream( self.env.from_collection([ (0, Vectors.dense(2.1, 3.1), 1.0, Vectors.sparse( 5, [3], [1.0])), (1, Vectors.dense(2.1, 3.1), 1.0, Vectors.sparse(5, [1, 2, 3, 4], [1.0, 2.0, 3.0, 4.0])), (2, None, None, None), ], type_info=Types.ROW_NAMED( ['id', 'vec', 'num', 'sparse_vec'], [ Types.INT(), DenseVectorTypeInfo(), Types.DOUBLE(), SparseVectorTypeInfo() ]))) self.expected_output_data_1 = Vectors.sparse(8, [0, 1, 2, 6], [2.1, 3.1, 1.0, 1.0]) self.expected_output_data_2 = Vectors.dense(2.1, 3.1, 1.0, 0.0, 1.0, 2.0, 3.0, 4.0)
def setUp(self): super(KMeansTest, self).setUp() self.data_table = self.t_env.from_data_stream( self.env.from_collection( [ (Vectors.dense([0.0, 0.0]), ), (Vectors.dense([0.0, 0.3]), ), (Vectors.dense([0.3, 3.0]), ), (Vectors.dense([9.0, 0.0]), ), (Vectors.dense([9.0, 0.6]), ), (Vectors.dense([9.6, 0.0]), ), ], type_info=Types.ROW_NAMED(['features'], [DenseVectorTypeInfo()]))) self.expected_groups = [{ DenseVector([0.0, 0.3]), DenseVector([0.3, 3.0]), DenseVector([0.0, 0.0]) }, { DenseVector([9.6, 0.0]), DenseVector([9.0, 0.0]), DenseVector([9.0, 0.6]) }]
def setUp(self): super(BinaryClassificationEvaluatorTest, self).setUp() self.input_data_table = self.t_env.from_data_stream( self.env.from_collection([ (1.0, Vectors.dense(0.1, 0.9)), (1.0, Vectors.dense(0.2, 0.8)), (1.0, Vectors.dense(0.3, 0.7)), (0.0, Vectors.dense(0.25, 0.75)), (0.0, Vectors.dense(0.4, 0.6)), (1.0, Vectors.dense(0.35, 0.65)), (1.0, Vectors.dense(0.45, 0.55)), (0.0, Vectors.dense(0.6, 0.4)), (0.0, Vectors.dense(0.7, 0.3)), (1.0, Vectors.dense(0.65, 0.35)), (0.0, Vectors.dense(0.8, 0.2)), (1.0, Vectors.dense(0.9, 0.1)) ], type_info=Types.ROW_NAMED( ['label', 'rawPrediction'], [Types.DOUBLE(), DenseVectorTypeInfo()])) ) self.input_data_table_score = self.t_env.from_data_stream( self.env.from_collection([ (1, 0.9), (1, 0.8), (1, 0.7), (0, 0.75), (0, 0.6), (1, 0.65), (1, 0.55), (0, 0.4), (0, 0.3), (1, 0.35), (0, 0.2), (1, 0.1) ], type_info=Types.ROW_NAMED( ['label', 'rawPrediction'], [Types.INT(), Types.DOUBLE()])) ) self.input_data_table_with_multi_score = self.t_env.from_data_stream( self.env.from_collection([ (1.0, Vectors.dense(0.1, 0.9)), (1.0, Vectors.dense(0.1, 0.9)), (1.0, Vectors.dense(0.1, 0.9)), (0.0, Vectors.dense(0.25, 0.75)), (0.0, Vectors.dense(0.4, 0.6)), (1.0, Vectors.dense(0.1, 0.9)), (1.0, Vectors.dense(0.1, 0.9)), (0.0, Vectors.dense(0.6, 0.4)), (0.0, Vectors.dense(0.7, 0.3)), (1.0, Vectors.dense(0.1, 0.9)), (0.0, Vectors.dense(0.8, 0.2)), (1.0, Vectors.dense(0.9, 0.1)) ], type_info=Types.ROW_NAMED( ['label', 'rawPrediction'], [Types.DOUBLE(), DenseVectorTypeInfo()])) ) self.input_data_table_with_weight = self.t_env.from_data_stream( self.env.from_collection([ (1.0, Vectors.dense(0.1, 0.9), 0.8), (1.0, Vectors.dense(0.1, 0.9), 0.7), (1.0, Vectors.dense(0.1, 0.9), 0.5), (0.0, Vectors.dense(0.25, 0.75), 1.2), (0.0, Vectors.dense(0.4, 0.6), 1.3), (1.0, Vectors.dense(0.1, 0.9), 1.5), (1.0, Vectors.dense(0.1, 0.9), 1.4), (0.0, Vectors.dense(0.6, 0.4), 0.3), (0.0, Vectors.dense(0.7, 0.3), 0.5), (1.0, Vectors.dense(0.1, 0.9), 1.9), (0.0, Vectors.dense(0.8, 0.2), 1.2), (1.0, Vectors.dense(0.9, 0.1), 1.0) ], type_info=Types.ROW_NAMED( ['label', 'rawPrediction', 'weight'], [Types.DOUBLE(), DenseVectorTypeInfo(), Types.DOUBLE()])) ) self.expected_data = [0.7691481137909708, 0.3714285714285714, 0.6571428571428571] self.expected_data_m = [0.8571428571428571, 0.9377705627705628, 0.8571428571428571, 0.6488095238095237] self.expected_data_w = 0.8911680911680911 self.eps = 1e-5
def setUp(self): super(MinMaxScalerTest, self).setUp() self.train_data = self.t_env.from_data_stream( self.env.from_collection([ (Vectors.dense([0.0, 3.0]), ), (Vectors.dense([2.1, 0.0]), ), (Vectors.dense([4.1, 5.1]), ), (Vectors.dense([6.1, 8.1]), ), (Vectors.dense([200., 400.]), ), ], type_info=Types.ROW_NAMED( ['input'], [DenseVectorTypeInfo()]))) self.predict_data = self.t_env.from_data_stream( self.env.from_collection([ (Vectors.dense([150.0, 90.0]), ), (Vectors.dense([50.0, 40.0]), ), (Vectors.dense([100.0, 50.0]), ), ], type_info=Types.ROW_NAMED( ['input'], [DenseVectorTypeInfo()]))) self.expected_data = [ Vectors.dense(0.25, 0.1), Vectors.dense(0.5, 0.125), Vectors.dense(0.75, 0.225) ]
def setUp(self): super(StandardScalerTest, self).setUp() self.dense_input = self.t_env.from_data_stream( self.env.from_collection([ (Vectors.dense(-2.5, 9.0, 1.0), ), (Vectors.dense(1.4, -5.0, 1.0), ), (Vectors.dense(2.0, -1.0, -2.0), ), ], type_info=Types.ROW_NAMED( ['input'], [DenseVectorTypeInfo()]))) self.expected_res_with_mean = [ Vectors.dense(-2.8, 8.0, 1.0), Vectors.dense(1.1, -6.0, 1.0), Vectors.dense(1.7, -2.0, -2.0) ] self.expected_res_with_std = [ Vectors.dense(-1.0231819, 1.2480754, 0.5773502), Vectors.dense(0.5729819, -0.6933752, 0.5773503), Vectors.dense(0.8185455, -0.1386750, -1.1547005) ] self.expected_res_with_mean_and_std = [ Vectors.dense(-1.1459637, 1.1094004, 0.5773503), Vectors.dense(0.45020003, -0.8320503, 0.5773503), Vectors.dense(0.69576368, -0.2773501, -1.1547005) ] self.expected_mean = [0.3, 1.0, 0.0] self.expected_std = [2.4433583, 7.2111026, 1.7320508]
def setUp(self): super(KNNTest, self).setUp() self.train_data = self.t_env.from_data_stream( self.env.from_collection([ (Vectors.dense([2.0, 3.0]), 1.0), (Vectors.dense([2.1, 3.1]), 1.0), (Vectors.dense([200.1, 300.1]), 2.0), (Vectors.dense([200.2, 300.2]), 2.0), (Vectors.dense([200.3, 300.3]), 2.0), (Vectors.dense([200.4, 300.4]), 2.0), (Vectors.dense([200.4, 300.4]), 2.0), (Vectors.dense([200.6, 300.6]), 2.0), (Vectors.dense([2.1, 3.1]), 1.0), (Vectors.dense([2.1, 3.1]), 1.0), (Vectors.dense([2.1, 3.1]), 1.0), (Vectors.dense([2.1, 3.1]), 1.0), (Vectors.dense([2.3, 3.2]), 1.0), (Vectors.dense([2.3, 3.2]), 1.0), (Vectors.dense([2.8, 3.2]), 3.0), (Vectors.dense([300., 3.2]), 4.0), (Vectors.dense([2.2, 3.2]), 1.0), (Vectors.dense([2.4, 3.2]), 5.0), (Vectors.dense([2.5, 3.2]), 5.0), (Vectors.dense([2.5, 3.2]), 5.0), (Vectors.dense([2.1, 3.1]), 1.0) ], type_info=Types.ROW_NAMED( ['features', 'label'], [DenseVectorTypeInfo(), Types.DOUBLE()]))) self.predict_data = self.t_env.from_data_stream( self.env.from_collection([ (Vectors.dense([4.0, 4.1]), 5.0), (Vectors.dense([300, 42]), 2.0), ], type_info=Types.ROW_NAMED( ['features', 'label'], [DenseVectorTypeInfo(), Types.DOUBLE()])))
def setUp(self): super(NaiveBayesTest, self).setUp() self.env.set_parallelism(1) self.train_data = self.t_env.from_data_stream( self.env.from_collection( [ (Vectors.dense([0, 0.]), 11.), (Vectors.dense([1, 0]), 10.), (Vectors.dense([1, 1.]), 10.), ], type_info=Types.ROW_NAMED( ['features', 'label'], [DenseVectorTypeInfo(), Types.DOUBLE()]))) self.predict_data = self.t_env.from_data_stream( self.env.from_collection( [ (Vectors.dense([0, 1.]), ), (Vectors.dense([0, 0.]), ), (Vectors.dense([1, 0]), ), (Vectors.dense([1, 1.]), ), ], type_info=Types.ROW_NAMED(['features'], [DenseVectorTypeInfo()]))) self.expected_output = { Vectors.dense([0, 1.]): 11., Vectors.dense([0, 0.]): 11., Vectors.dense([1, 0.]): 10., Vectors.dense([1, 1.]): 10., } self.estimator = NaiveBayes() \ .set_smoothing(1.0) \ .set_features_col('features') \ .set_label_col('label') \ .set_prediction_col('prediction') \ .set_model_type('multinomial') # type: NaiveBayes