def test_bucket_column(self):
desc = self.new_field_desc()
nc = fc.NumericColumn(desc)
boundaries = [-10.5, 20]
bc = fc.BucketColumn(nc, boundaries)
self.assertEqual(bc.source_column, nc)
self.assertEqual(bc.boundaries, boundaries)
self.assertEqual(bc.num_class(), len(boundaries) + 1)
self.assertEqual(len(bc.get_field_desc()), 1)
self.assertEqual(bc.get_field_desc()[0].to_json(), desc.to_json())
d = fc.FeatureColumn.to_dict(bc)
self.assertEqual(d["type"], "BucketColumn")
self.assertEqual(d["boundaries"], boundaries)
self.assertEqual(d["source_column"]["type"], "NumericColumn")
self.assertEqual(d["source_column"]["field_desc"], desc.to_dict())
self.check_from_dict(bc)
bc = bc.new_feature_column_from(desc)
self.assertTrue(isinstance(bc, fc.BucketColumn))
self.assertEqual(bc.boundaries, boundaries)
self.assertEqual(bc.num_class(), len(boundaries) + 1)
self.assertEqual(len(bc.get_field_desc()), 1)
self.assertEqual(bc.get_field_desc()[0].to_json(), desc.to_json())
d = fc.FeatureColumn.to_dict(bc)
self.assertEqual(d["type"], "BucketColumn")
self.assertEqual(d["boundaries"], boundaries)
self.assertEqual(d["source_column"]["type"], "NumericColumn")
self.assertEqual(d["source_column"]["field_desc"], desc.to_dict())
self.check_from_dict(bc)
def test_cross_column(self):
desc = self.new_field_desc()
nc = fc.NumericColumn(desc)
hash_bucket_size = 1024
cc = fc.CrossColumn([nc, 'cross_feature_2'], hash_bucket_size)
self.assertEqual(cc.num_class(), hash_bucket_size)
descs = cc.get_field_desc()
self.assertEqual(len(descs), 2)
self.assertEqual(descs[0].to_json(), desc.to_json())
self.assertEqual(descs[1].name, 'cross_feature_2')
def test_numeric_column(self):
desc1 = self.new_field_desc()
desc2 = self.new_field_desc()
nc1 = fc.NumericColumn(desc1)
nc2 = nc1.new_feature_column_from(desc2)
self.assertTrue(isinstance(nc2, fc.NumericColumn))
self.assertEqual(len(nc1.get_field_desc()), 1)
self.assertEqual(len(nc2.get_field_desc()), 1)
self.assertEqual(nc1.get_field_desc()[0].to_json(),
nc2.get_field_desc()[0].to_json())
def test_bucket_column(self):
desc = self.new_field_desc()
nc = fc.NumericColumn(desc)
boundaries = [-10.5, 20]
bc = fc.BucketColumn(nc, boundaries)
self.assertEqual(bc.source_column, nc)
self.assertEqual(bc.boundaries, boundaries)
self.assertEqual(bc.num_class(), len(boundaries) + 1)
self.assertEqual(len(bc.get_field_desc()), 1)
self.assertEqual(bc.get_field_desc()[0].to_json(), desc.to_json())
bc = bc.new_feature_column_from(desc)
self.assertTrue(isinstance(bc, fc.BucketColumn))
self.assertEqual(bc.boundaries, boundaries)
self.assertEqual(bc.num_class(), len(boundaries) + 1)
self.assertEqual(len(bc.get_field_desc()), 1)
self.assertEqual(bc.get_field_desc()[0].to_json(), desc.to_json())
def test_cross_column(self):
desc = self.new_field_desc()
nc = fc.NumericColumn(desc)
hash_bucket_size = 1024
cc = fc.CrossColumn([nc, 'cross_feature_2'], hash_bucket_size)
self.assertEqual(cc.num_class(), hash_bucket_size)
descs = cc.get_field_desc()
self.assertEqual(len(descs), 2)
self.assertEqual(descs[0].to_json(), desc.to_json())
self.assertEqual(descs[1].name, 'cross_feature_2')
d = fc.FeatureColumn.to_dict(cc)
self.assertEqual(d["type"], "CrossColumn")
keys = d["keys"]
self.assertEqual(len(keys), 2)
self.assertEqual(keys[0]["type"], "NumericColumn")
self.assertEqual(keys[0]["field_desc"], desc.to_dict())
self.assertEqual(keys[1], "cross_feature_2")
self.assertEqual(d["hash_bucket_size"], hash_bucket_size)
self.check_from_dict(cc)
def test_numeric_column(self):
desc1 = self.new_field_desc()
desc2 = self.new_field_desc()
nc1 = fc.NumericColumn(desc1)
nc2 = nc1.new_feature_column_from(desc2)
self.assertTrue(isinstance(nc2, fc.NumericColumn))
self.assertEqual(len(nc1.get_field_desc()), 1)
self.assertEqual(len(nc2.get_field_desc()), 1)
self.assertEqual(nc1.get_field_desc()[0].to_json(),
nc2.get_field_desc()[0].to_json())
d1 = fc.FeatureColumn.to_dict(nc1)
self.assertEqual(d1["type"], "NumericColumn")
self.assertEqual(d1["field_desc"], desc1.to_dict())
self.check_from_dict(nc1)
d2 = fc.FeatureColumn.to_dict(nc2)
self.assertEqual(d2["type"], "NumericColumn")
self.assertEqual(d2["field_desc"], desc2.to_dict())
self.check_from_dict(nc2)
"role_arn=arn/pai2osstestproject&host=host' "
r'''-Dcluster="{\"ps\": {\"count\": 1, \"cpu\": 200, \"gpu\": 0}'''
r''', \"worker\": {\"count\": 5, \"cpu\": 400, \"gpu\": 0}}"''')
self.assertEqual(expected, cmd)
del os.environ["SQLFLOW_OSS_CHECKPOINT_CONFIG"]
iris_feature_column_names = [
"sepal_length",
"sepal_width",
"petal_length",
"petal_width",
]
feature_column_map = {
"feature_columns": [fc.NumericColumn(fd.FieldDesc(name="sepal_length"))]
}
label_column = fc.NumericColumn(fd.FieldDesc(name="class"))
@unittest.skipUnless(testing.get_driver() == "maxcompute"
and testing.get_submitter() == "pai",
"skip non PAI tests")
class SubmitPAITrainTask(TestCase):
def test_submit_pai_train_task(self):
model_params = dict()
model_params["hidden_units"] = [10, 20]
model_params["n_classes"] = 3
original_sql = """
SELECT * FROM alifin_jtest_dev.sqlflow_test_iris_train
