def setUpClass(cls):
cls.features = Features()
cls.features.append(DefFeature("A", str, 1))
A, X = list(zip(*DATA))
cls.records = [
Record(str(i), data={"features": {
"A": A[i],
"X": X[i]
}}) for i in range(0, len(X))
]
cls.sources = Sources(
MemorySource(MemorySourceConfig(records=cls.records)))
cls.model_dir = tempfile.TemporaryDirectory()
cls.model = TextClassificationModel(
TextClassifierConfig(
directory=cls.model_dir.name,
classifications=[0, 1],
features=cls.features,
predict=DefFeature("X", int, 1),
add_layers=True,
layers=[
"Dense(units = 120, activation='relu')",
"Dense(units = 64, activation=relu)",
"Dense(units = 2, activation='softmax')",
],
model_path=
"https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim-with-oov/1",
epochs=30,
))
async def test_model(self):
with tempfile.TemporaryDirectory() as tempdir, patch.object(
Model, "load", new=model_load
):
config = parse_unknown(
"--model-directory",
tempdir,
"--model-features",
"Years:int:1",
"Experiance:int:1",
)
async with self.post(
"/configure/model/fake/salary", json=config
) as r:
self.assertEqual(await r.json(), OK)
self.assertIn("salary", self.cli.app["models"])
self.assertEqual(
self.cli.app["models"]["salary"].config,
FakeModelConfig(
directory=tempdir,
features=Features(
DefFeature("Years", int, 1),
DefFeature("Experiance", int, 1),
),
),
)
with self.subTest(context="salaryctx"):
# Create the context
async with self.get(
"/context/model/salary/salaryctx"
) as r:
self.assertEqual(await r.json(), OK)
self.assertIn(
"salaryctx", self.cli.app["model_contexts"]
)
async def test_model(self):
with tempfile.TemporaryDirectory() as tempdir, patch.object(
Model, "load", new=model_load):
config = parse_unknown("--model-directory", tempdir)
async with self.post("/configure/model/fake/salary",
json=config) as r:
self.assertEqual(await r.json(), OK)
self.assertIn("salary", self.cli.app["models"])
self.assertEqual(
self.cli.app["models"]["salary"].config,
FakeModelConfig(directory=tempdir),
)
with self.subTest(context="salaryctx"):
# Define the features
features = Features(
DefFeature("Years", int, 1),
DefFeature("Experiance", int, 1),
)
exported_features = features.export()
# Check that we can send shorthand version of feature_def
for name, feature_def in exported_features.items():
del feature_def["name"]
# Create the context
async with self.post(
"/context/model/salary/salaryctx",
json=exported_features,
) as r:
self.assertEqual(await r.json(), OK)
self.assertIn("salaryctx",
self.cli.app["model_contexts"])
self.assertEqual(
self.cli.app["model_contexts"]
["salaryctx"].features.export(),
features.export(),
)
def setUpClass(cls):
cls.model_dir = tempfile.TemporaryDirectory()
cls.feature1 = Feature_1()
cls.feature2 = Feature_2()
cls.features = Features(cls.feature1, cls.feature2)
cls.model = DNNRegressionModel(
DNNRegressionModelConfig(
directory=cls.model_dir.name,
steps=1000,
epochs=40,
hidden=[50, 20, 10],
predict=DefFeature("TARGET", float, 1),
features=cls.features,
))
# Generating data f(x1,x2) = 2*x1 + 3*x2
_n_data = 2000
_temp_data = np.random.rand(2, _n_data)
cls.repos = [
Repo(
"x" + str(random.random()),
data={
"features": {
cls.feature1.NAME: float(_temp_data[0][i]),
cls.feature2.NAME: float(_temp_data[1][i]),
"TARGET": 2 * _temp_data[0][i] + 3 * _temp_data[1][i],
}
},
) for i in range(0, _n_data)
]
cls.sources = Sources(MemorySource(
MemorySourceConfig(repos=cls.repos)))
def setUpClass(cls):
cls.model_dir = tempfile.TemporaryDirectory()
cls.feature = StartsWithA()
cls.features = Features(cls.feature)
cls.records = [
Record(
"a" + str(random.random()),
data={"features": {
cls.feature.NAME: 1,
"string": "a"
}},
) for _ in range(0, 1000)
]
cls.records += [
Record(
"b" + str(random.random()),
data={"features": {
cls.feature.NAME: 0,
"string": "not a"
}},
) for _ in range(0, 1000)
]
cls.sources = Sources(
MemorySource(MemorySourceConfig(records=cls.records)))
cls.model = DNNClassifierModel(
DNNClassifierModelConfig(
directory=cls.model_dir.name,
steps=1000,
epochs=40,
hidden=[50, 20, 10],
predict=DefFeature("string", str, 1),
classifications=["a", "not a"],
clstype=str,
features=cls.features,
))
def setUpClass(cls):
cls.model_dir = tempfile.TemporaryDirectory()
cls.feature = DefFeature("X", float, 1)
cls.features = Features(cls.feature)
X, Y = list(zip(*FEATURE_DATA))
cls.repos = [
Repo(str(i), data={"features": {"X": X[i], "Y": Y[i]}})
for i in range(0, len(Y))
]
cls.sources = Sources(
MemorySource(MemorySourceConfig(repos=cls.repos))
)
cls.model = SLR(
SLRConfig(
directory=cls.model_dir.name,
predict=DefFeature("Y", float, 1),
features=cls.features,
)
)
async def _add_memory_source(self):
self.features = Features(DefFeature("by_ten", int, 1))
async with MemorySource(
MemorySourceConfig(repos=[
Repo(str(i), data={"features": {
"by_ten": i * 10
}}) for i in range(0, self.num_repos)
])) as source:
self.source = self.cli.app["sources"][self.slabel] = source
async with source() as sctx:
self.sctx = self.cli.app["source_contexts"][self.slabel] = sctx
yield
def setUp(self):
super().setUp()
self.repo_keys = {'add 40 and 2': 42, 'multiply 42 and 10': 420}
self.repos = list(map(Repo, self.repo_keys.keys()))
self.sources = Sources(RepoSource(*self.repos))
self.features = Features(DefFeature('string_calculator', int, 1))
self.cli = OperationsAll(ops=OPERATIONS,
opimpn_memory_opimps=OPIMPS,
repo_def='calc_string',
output_specs=[(
['result'],
'get_single_spec',
)],
remap=[('get_single', 'result',
'string_calculator')],
sources=self.sources,
features=self.features)
def setUpClass(cls):
cls.model_dir = tempfile.TemporaryDirectory()
cls.model = cls.MODEL(
cls.MODEL_CONFIG(directory=cls.model_dir.name, predict="X"))
cls.features = Features()
if cls.MODEL_TYPE is "CLASSIFICATION":
cls.features.append(DefFeature("A", float, 1))
cls.features.append(DefFeature("B", float, 1))
cls.features.append(DefFeature("C", float, 1))
cls.features.append(DefFeature("D", float, 1))
cls.features.append(DefFeature("E", float, 1))
cls.features.append(DefFeature("F", float, 1))
cls.features.append(DefFeature("G", float, 1))
cls.features.append(DefFeature("H", float, 1))
cls.features.append(DefFeature("I", float, 1))
A, B, C, D, E, F, G, H, I, X = list(
zip(*FEATURE_DATA_CLASSIFICATION))
cls.repos = [
Repo(
str(i),
data={
"features": {
"A": A[i],
"B": B[i],
"C": C[i],
"D": D[i],
"E": E[i],
"F": F[i],
"G": G[i],
"H": H[i],
"I": I[i],
"X": X[i],
}
},
) for i in range(0, len(A))
]
elif cls.MODEL_TYPE is "REGRESSION":
cls.features.append(DefFeature("A", float, 1))
cls.features.append(DefFeature("B", float, 1))
cls.features.append(DefFeature("C", float, 1))
A, B, C, X = list(zip(*FEATURE_DATA_REGRESSION))
cls.repos = [
Repo(
str(i),
data={
"features": {
"A": A[i],
"B": B[i],
"C": C[i],
"X": X[i],
}
},
) for i in range(0, len(A))
]
cls.sources = Sources(MemorySource(
MemorySourceConfig(repos=cls.repos)))
def setUpClass(cls):
cls.model_dir = tempfile.TemporaryDirectory()
cls.features = Features()
if cls.MODEL_TYPE is "CLASSIFICATION":
cls.features.append(DefFeature("A", float, 1))
cls.features.append(DefFeature("B", float, 1))
cls.features.append(DefFeature("C", float, 1))
cls.features.append(DefFeature("D", float, 1))
cls.features.append(DefFeature("E", float, 1))
cls.features.append(DefFeature("F", float, 1))
cls.features.append(DefFeature("G", float, 1))
cls.features.append(DefFeature("H", float, 1))
cls.features.append(DefFeature("I", float, 1))
A, B, C, D, E, F, G, H, I, X = list(
zip(*FEATURE_DATA_CLASSIFICATION))
cls.repos = [
Repo(
str(i),
data={
"features": {
"A": A[i],
"B": B[i],
"C": C[i],
"D": D[i],
"E": E[i],
"F": F[i],
"G": G[i],
"H": H[i],
"I": I[i],
"X": X[i],
}
},
) for i in range(0, len(A))
]
elif cls.MODEL_TYPE is "REGRESSION":
cls.features.append(DefFeature("A", float, 1))
cls.features.append(DefFeature("B", float, 1))
cls.features.append(DefFeature("C", float, 1))
A, B, C, X = list(zip(*FEATURE_DATA_REGRESSION))
cls.repos = [
Repo(
str(i),
data={
"features": {
"A": A[i],
"B": B[i],
"C": C[i],
"X": X[i],
}
},
) for i in range(0, len(A))
]
elif cls.MODEL_TYPE is "CLUSTERING":
cls.features.append(DefFeature("A", float, 1))
cls.features.append(DefFeature("B", float, 1))
cls.features.append(DefFeature("C", float, 1))
cls.features.append(DefFeature("D", float, 1))
A, B, C, D, X = list(zip(*FEATURE_DATA_CLUSTERING))
cls.repos = [
Repo(
str(i),
data={
"features": {
"A": A[i],
"B": B[i],
"C": C[i],
"D": D[i],
"X": X[i],
}
},
) for i in range(0, len(A))
]
cls.sources = Sources(MemorySource(
MemorySourceConfig(repos=cls.repos)))
properties = {
"directory": cls.model_dir.name,
"features": cls.features,
}
config_fields = dict()
estimator_type = cls.MODEL.SCIKIT_MODEL._estimator_type
if estimator_type in supervised_estimators:
config_fields["predict"] = DefFeature("X", float, 1)
elif estimator_type in unsupervised_estimators:
if cls.TRUE_CLSTR_PRESENT:
config_fields["tcluster"] = DefFeature("X", float, 1)
cls.model = cls.MODEL(
cls.MODEL_CONFIG(**{
**properties,
**config_fields
}))
def test_deffeature(self):
feature = DefFeature("test", float, 10)
self.assertEqual(feature.NAME, "test")
self.assertEqual(feature.dtype(), float)
self.assertEqual(feature.length(), 10)
def setUpClass(cls):
cls.model_dir = tempfile.TemporaryDirectory()
cls.features = Features()
cls.features.append(DefFeature("A", float, 1))
cls.features.append(DefFeature("B", float, 1))
cls.features.append(DefFeature("C", float, 1))
cls.features.append(DefFeature("D", float, 1))
cls.features.append(DefFeature("E", float, 1))
cls.features.append(DefFeature("F", float, 1))
cls.features.append(DefFeature("G", int, 1))
cls.features.append(DefFeature("H", int, 1))
A, B, C, D, E, F, G, H, X = list(zip(*DATA))
cls.records = [
Record(
str(i),
data={
"features": {
"A": A[i],
"B": B[i],
"C": C[i],
"D": D[i],
"E": E[i],
"F": F[i],
"G": G[i],
"H": H[i],
"X": X[i],
}
},
) for i in range(0, len(A))
]
cls.sources = Sources(
MemorySource(MemorySourceConfig(records=cls.records)))
cls.model = VWModel(
VWConfig(
directory=cls.model_dir.name,
features=cls.features,
predict=DefFeature("X", float, 1),
# A and B will be namespace n1
# A and C will be in namespace n2
namespace=["n1_A_B", "n2_A_C"],
importance=DefFeature("H", int, 1),
tag=DefFeature("G", int, 1),
task="regression",
convert_to_vw=True,
vwcmd=[
"l2",
"0.1",
"loss_function",
"squared",
"passes",
"10",
],
))
