def setUpClass(cls): cls.features = Features() cls.features.append(DefFeature("A", str, 1)) A, X = list(zip(*DATA)) cls.records = [ Record(str(i), data={"features": { "A": A[i], "X": X[i] }}) for i in range(0, len(X)) ] cls.sources = Sources( MemorySource(MemorySourceConfig(records=cls.records))) cls.model_dir = tempfile.TemporaryDirectory() cls.model = TextClassificationModel( TextClassifierConfig( directory=cls.model_dir.name, classifications=[0, 1], features=cls.features, predict=DefFeature("X", int, 1), add_layers=True, layers=[ "Dense(units = 120, activation='relu')", "Dense(units = 64, activation=relu)", "Dense(units = 2, activation='softmax')", ], model_path= "https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim-with-oov/1", epochs=30, ))
async def test_model(self): with tempfile.TemporaryDirectory() as tempdir, patch.object( Model, "load", new=model_load ): config = parse_unknown( "--model-directory", tempdir, "--model-features", "Years:int:1", "Experiance:int:1", ) async with self.post( "/configure/model/fake/salary", json=config ) as r: self.assertEqual(await r.json(), OK) self.assertIn("salary", self.cli.app["models"]) self.assertEqual( self.cli.app["models"]["salary"].config, FakeModelConfig( directory=tempdir, features=Features( DefFeature("Years", int, 1), DefFeature("Experiance", int, 1), ), ), ) with self.subTest(context="salaryctx"): # Create the context async with self.get( "/context/model/salary/salaryctx" ) as r: self.assertEqual(await r.json(), OK) self.assertIn( "salaryctx", self.cli.app["model_contexts"] )
async def test_model(self): with tempfile.TemporaryDirectory() as tempdir, patch.object( Model, "load", new=model_load): config = parse_unknown("--model-directory", tempdir) async with self.post("/configure/model/fake/salary", json=config) as r: self.assertEqual(await r.json(), OK) self.assertIn("salary", self.cli.app["models"]) self.assertEqual( self.cli.app["models"]["salary"].config, FakeModelConfig(directory=tempdir), ) with self.subTest(context="salaryctx"): # Define the features features = Features( DefFeature("Years", int, 1), DefFeature("Experiance", int, 1), ) exported_features = features.export() # Check that we can send shorthand version of feature_def for name, feature_def in exported_features.items(): del feature_def["name"] # Create the context async with self.post( "/context/model/salary/salaryctx", json=exported_features, ) as r: self.assertEqual(await r.json(), OK) self.assertIn("salaryctx", self.cli.app["model_contexts"]) self.assertEqual( self.cli.app["model_contexts"] ["salaryctx"].features.export(), features.export(), )
def setUpClass(cls): cls.model_dir = tempfile.TemporaryDirectory() cls.feature1 = Feature_1() cls.feature2 = Feature_2() cls.features = Features(cls.feature1, cls.feature2) cls.model = DNNRegressionModel( DNNRegressionModelConfig( directory=cls.model_dir.name, steps=1000, epochs=40, hidden=[50, 20, 10], predict=DefFeature("TARGET", float, 1), features=cls.features, )) # Generating data f(x1,x2) = 2*x1 + 3*x2 _n_data = 2000 _temp_data = np.random.rand(2, _n_data) cls.repos = [ Repo( "x" + str(random.random()), data={ "features": { cls.feature1.NAME: float(_temp_data[0][i]), cls.feature2.NAME: float(_temp_data[1][i]), "TARGET": 2 * _temp_data[0][i] + 3 * _temp_data[1][i], } }, ) for i in range(0, _n_data) ] cls.sources = Sources(MemorySource( MemorySourceConfig(repos=cls.repos)))
def setUpClass(cls): cls.model_dir = tempfile.TemporaryDirectory() cls.feature = StartsWithA() cls.features = Features(cls.feature) cls.records = [ Record( "a" + str(random.random()), data={"features": { cls.feature.NAME: 1, "string": "a" }}, ) for _ in range(0, 1000) ] cls.records += [ Record( "b" + str(random.random()), data={"features": { cls.feature.NAME: 0, "string": "not a" }}, ) for _ in range(0, 1000) ] cls.sources = Sources( MemorySource(MemorySourceConfig(records=cls.records))) cls.model = DNNClassifierModel( DNNClassifierModelConfig( directory=cls.model_dir.name, steps=1000, epochs=40, hidden=[50, 20, 10], predict=DefFeature("string", str, 1), classifications=["a", "not a"], clstype=str, features=cls.features, ))
def setUpClass(cls): cls.model_dir = tempfile.TemporaryDirectory() cls.feature = DefFeature("X", float, 1) cls.features = Features(cls.feature) X, Y = list(zip(*FEATURE_DATA)) cls.repos = [ Repo(str(i), data={"features": {"X": X[i], "Y": Y[i]}}) for i in range(0, len(Y)) ] cls.sources = Sources( MemorySource(MemorySourceConfig(repos=cls.repos)) ) cls.model = SLR( SLRConfig( directory=cls.model_dir.name, predict=DefFeature("Y", float, 1), features=cls.features, ) )
async def _add_memory_source(self): self.features = Features(DefFeature("by_ten", int, 1)) async with MemorySource( MemorySourceConfig(repos=[ Repo(str(i), data={"features": { "by_ten": i * 10 }}) for i in range(0, self.num_repos) ])) as source: self.source = self.cli.app["sources"][self.slabel] = source async with source() as sctx: self.sctx = self.cli.app["source_contexts"][self.slabel] = sctx yield
def setUp(self): super().setUp() self.repo_keys = {'add 40 and 2': 42, 'multiply 42 and 10': 420} self.repos = list(map(Repo, self.repo_keys.keys())) self.sources = Sources(RepoSource(*self.repos)) self.features = Features(DefFeature('string_calculator', int, 1)) self.cli = OperationsAll(ops=OPERATIONS, opimpn_memory_opimps=OPIMPS, repo_def='calc_string', output_specs=[( ['result'], 'get_single_spec', )], remap=[('get_single', 'result', 'string_calculator')], sources=self.sources, features=self.features)
def setUpClass(cls): cls.model_dir = tempfile.TemporaryDirectory() cls.model = cls.MODEL( cls.MODEL_CONFIG(directory=cls.model_dir.name, predict="X")) cls.features = Features() if cls.MODEL_TYPE is "CLASSIFICATION": cls.features.append(DefFeature("A", float, 1)) cls.features.append(DefFeature("B", float, 1)) cls.features.append(DefFeature("C", float, 1)) cls.features.append(DefFeature("D", float, 1)) cls.features.append(DefFeature("E", float, 1)) cls.features.append(DefFeature("F", float, 1)) cls.features.append(DefFeature("G", float, 1)) cls.features.append(DefFeature("H", float, 1)) cls.features.append(DefFeature("I", float, 1)) A, B, C, D, E, F, G, H, I, X = list( zip(*FEATURE_DATA_CLASSIFICATION)) cls.repos = [ Repo( str(i), data={ "features": { "A": A[i], "B": B[i], "C": C[i], "D": D[i], "E": E[i], "F": F[i], "G": G[i], "H": H[i], "I": I[i], "X": X[i], } }, ) for i in range(0, len(A)) ] elif cls.MODEL_TYPE is "REGRESSION": cls.features.append(DefFeature("A", float, 1)) cls.features.append(DefFeature("B", float, 1)) cls.features.append(DefFeature("C", float, 1)) A, B, C, X = list(zip(*FEATURE_DATA_REGRESSION)) cls.repos = [ Repo( str(i), data={ "features": { "A": A[i], "B": B[i], "C": C[i], "X": X[i], } }, ) for i in range(0, len(A)) ] cls.sources = Sources(MemorySource( MemorySourceConfig(repos=cls.repos)))
def setUpClass(cls): cls.model_dir = tempfile.TemporaryDirectory() cls.features = Features() if cls.MODEL_TYPE is "CLASSIFICATION": cls.features.append(DefFeature("A", float, 1)) cls.features.append(DefFeature("B", float, 1)) cls.features.append(DefFeature("C", float, 1)) cls.features.append(DefFeature("D", float, 1)) cls.features.append(DefFeature("E", float, 1)) cls.features.append(DefFeature("F", float, 1)) cls.features.append(DefFeature("G", float, 1)) cls.features.append(DefFeature("H", float, 1)) cls.features.append(DefFeature("I", float, 1)) A, B, C, D, E, F, G, H, I, X = list( zip(*FEATURE_DATA_CLASSIFICATION)) cls.repos = [ Repo( str(i), data={ "features": { "A": A[i], "B": B[i], "C": C[i], "D": D[i], "E": E[i], "F": F[i], "G": G[i], "H": H[i], "I": I[i], "X": X[i], } }, ) for i in range(0, len(A)) ] elif cls.MODEL_TYPE is "REGRESSION": cls.features.append(DefFeature("A", float, 1)) cls.features.append(DefFeature("B", float, 1)) cls.features.append(DefFeature("C", float, 1)) A, B, C, X = list(zip(*FEATURE_DATA_REGRESSION)) cls.repos = [ Repo( str(i), data={ "features": { "A": A[i], "B": B[i], "C": C[i], "X": X[i], } }, ) for i in range(0, len(A)) ] elif cls.MODEL_TYPE is "CLUSTERING": cls.features.append(DefFeature("A", float, 1)) cls.features.append(DefFeature("B", float, 1)) cls.features.append(DefFeature("C", float, 1)) cls.features.append(DefFeature("D", float, 1)) A, B, C, D, X = list(zip(*FEATURE_DATA_CLUSTERING)) cls.repos = [ Repo( str(i), data={ "features": { "A": A[i], "B": B[i], "C": C[i], "D": D[i], "X": X[i], } }, ) for i in range(0, len(A)) ] cls.sources = Sources(MemorySource( MemorySourceConfig(repos=cls.repos))) properties = { "directory": cls.model_dir.name, "features": cls.features, } config_fields = dict() estimator_type = cls.MODEL.SCIKIT_MODEL._estimator_type if estimator_type in supervised_estimators: config_fields["predict"] = DefFeature("X", float, 1) elif estimator_type in unsupervised_estimators: if cls.TRUE_CLSTR_PRESENT: config_fields["tcluster"] = DefFeature("X", float, 1) cls.model = cls.MODEL( cls.MODEL_CONFIG(**{ **properties, **config_fields }))
def test_deffeature(self): feature = DefFeature("test", float, 10) self.assertEqual(feature.NAME, "test") self.assertEqual(feature.dtype(), float) self.assertEqual(feature.length(), 10)
def setUpClass(cls): cls.model_dir = tempfile.TemporaryDirectory() cls.features = Features() cls.features.append(DefFeature("A", float, 1)) cls.features.append(DefFeature("B", float, 1)) cls.features.append(DefFeature("C", float, 1)) cls.features.append(DefFeature("D", float, 1)) cls.features.append(DefFeature("E", float, 1)) cls.features.append(DefFeature("F", float, 1)) cls.features.append(DefFeature("G", int, 1)) cls.features.append(DefFeature("H", int, 1)) A, B, C, D, E, F, G, H, X = list(zip(*DATA)) cls.records = [ Record( str(i), data={ "features": { "A": A[i], "B": B[i], "C": C[i], "D": D[i], "E": E[i], "F": F[i], "G": G[i], "H": H[i], "X": X[i], } }, ) for i in range(0, len(A)) ] cls.sources = Sources( MemorySource(MemorySourceConfig(records=cls.records))) cls.model = VWModel( VWConfig( directory=cls.model_dir.name, features=cls.features, predict=DefFeature("X", float, 1), # A and B will be namespace n1 # A and C will be in namespace n2 namespace=["n1_A_B", "n2_A_C"], importance=DefFeature("H", int, 1), tag=DefFeature("G", int, 1), task="regression", convert_to_vw=True, vwcmd=[ "l2", "0.1", "loss_function", "squared", "passes", "10", ], ))