def testCartMustBuildCategoricalCategorical(self):
random.seed(12345)
numpy.seterr(divide="ignore", invalid="ignore")
dataset = Dataset.fromIterable(((a, b, c) for (x, y, z, a, b, c) in TestProducerCart.data()), 100000, ("a", "b", "c"))
tree = TreeNode.fromWholeDataset(dataset, "c")
tree.splitMaxDepth(2)
doc = tree.pfaDocument({"type": "record", "name": "Datum", "fields": [{"name": "a", "type": "string"}, {"name": "b", "type": "string"}]}, "TreeNode")
# look(doc, maxDepth=8)
self.assertEqual(doc["cells"]["tree"]["init"]["field"], "a")
self.assertEqual(doc["cells"]["tree"]["init"]["value"], ["A0", "A1", "A2", "A3"])
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["field"], "b")
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["value"], ["B6", "B8"])
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["pass"]["string"], "C6")
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["fail"]["string"], "C3")
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["field"], "b")
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["value"], ["B0"])
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["pass"]["string"], "C0")
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["fail"]["string"], "C0")
engine, = PFAEngine.fromJson(doc)
self.assertEqual(engine.action({"a": "A1", "b": "B6"}), "C6")
self.assertEqual(engine.action({"a": "A1", "b": "B2"}), "C3")
self.assertEqual(engine.action({"a": "A5", "b": "B0"}), "C0")
self.assertEqual(engine.action({"a": "A5", "b": "B4"}), "C0")
doc = tree.pfaDocument(
{"type": "record", "name": "Datum", "fields": [{"name": "a", "type": "string"}, {"name": "b", "type": "string"}]},
"TreeNode",
nodeScores=True, datasetSize=True, predictandDistribution=True, predictandUnique=True, entropy=True, gain=True)
# look(doc, maxDepth=8)
engine, = PFAEngine.fromJson(doc)
def testCartMustBuildCategoricalCategorical(self):
random.seed(12345)
numpy.seterr(divide="ignore", invalid="ignore")
dataset = Dataset.fromIterable(((a, b, c) for (x, y, z, a, b, c) in TestProducerCart.data()), 100000, ("a", "b", "c"))
tree = TreeNode.fromWholeDataset(dataset, "c")
tree.splitMaxDepth(2)
doc = tree.pfaDocument({"type": "record", "name": "Datum", "fields": [{"name": "a", "type": "string"}, {"name": "b", "type": "string"}]}, "TreeNode")
# look(doc, maxDepth=8)
self.assertEqual(doc["cells"]["tree"]["init"]["field"], "a")
self.assertEqual(doc["cells"]["tree"]["init"]["value"], ["A0", "A1", "A2", "A3"])
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["field"], "b")
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["value"], ["B6", "B8"])
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["pass"]["string"], "C6")
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["fail"]["string"], "C3")
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["field"], "b")
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["value"], ["B0"])
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["pass"]["string"], "C0")
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["fail"]["string"], "C0")
engine, = PFAEngine.fromJson(doc)
self.assertEqual(engine.action({"a": "A1", "b": "B6"}), "C6")
self.assertEqual(engine.action({"a": "A1", "b": "B2"}), "C3")
self.assertEqual(engine.action({"a": "A5", "b": "B0"}), "C0")
self.assertEqual(engine.action({"a": "A5", "b": "B4"}), "C0")
doc = tree.pfaDocument(
{"type": "record", "name": "Datum", "fields": [{"name": "a", "type": "string"}, {"name": "b", "type": "string"}]},
"TreeNode",
nodeScores=True, datasetSize=True, predictandDistribution=True, predictandUnique=True, entropy=True, gain=True)
# look(doc, maxDepth=8)
engine, = PFAEngine.fromJson(doc)
def testCartMustBuildNumericalCategorical(self):
random.seed(12345)
numpy.seterr(divide="ignore", invalid="ignore")
dataset = Dataset.fromIterable(((x, y, c) for (x, y, z, a, b, c) in TestProducerCart.data()), 100000, ("x", "y", "c"))
tree = TreeNode.fromWholeDataset(dataset, "c")
tree.splitMaxDepth(2)
doc = tree.pfaDocument({"type": "record", "name": "Datum", "fields": [{"name": "x", "type": "double"}, {"name": "y", "type": "double"}]}, "TreeNode")
# look(doc, maxDepth=8)
self.assertEqual(doc["cells"]["tree"]["init"]["field"], "x")
self.assertAlmostEqual(doc["cells"]["tree"]["init"]["value"], 4.00, places=2)
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["field"], "y")
self.assertAlmostEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["value"], 6.00, places=2)
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["pass"]["string"], "C3")
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["fail"]["string"], "C6")
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["field"], "y")
self.assertAlmostEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["value"], 2.00, places=2)
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["pass"]["string"], "C0")
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["fail"]["string"], "C0")
engine, = PFAEngine.fromJson(doc)
self.assertEqual(engine.action({"x": 2.0, "y": 3.0}), "C3")
self.assertEqual(engine.action({"x": 2.0, "y": 8.0}), "C6")
self.assertEqual(engine.action({"x": 7.0, "y": 1.0}), "C0")
self.assertEqual(engine.action({"x": 7.0, "y": 5.0}), "C0")
doc = tree.pfaDocument(
{"type": "record", "name": "Datum", "fields": [{"name": "x", "type": "double"}, {"name": "y", "type": "double"}]},
"TreeNode",
nodeScores=True, datasetSize=True, predictandDistribution=True, predictandUnique=True, entropy=True, gain=True)
# look(doc, maxDepth=8)
engine, = PFAEngine.fromJson(doc)
def testCartMustBuildNumericalCategorical(self):
random.seed(12345)
numpy.seterr(divide="ignore", invalid="ignore")
dataset = Dataset.fromIterable(((x, y, c) for (x, y, z, a, b, c) in TestProducerCart.data()), 100000, ("x", "y", "c"))
tree = TreeNode.fromWholeDataset(dataset, "c")
tree.splitMaxDepth(2)
doc = tree.pfaDocument({"type": "record", "name": "Datum", "fields": [{"name": "x", "type": "double"}, {"name": "y", "type": "double"}]}, "TreeNode")
# look(doc, maxDepth=8)
self.assertEqual(doc["cells"]["tree"]["init"]["field"], "x")
self.assertAlmostEqual(doc["cells"]["tree"]["init"]["value"], 4.00, places=2)
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["field"], "y")
self.assertAlmostEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["value"], 6.00, places=2)
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["pass"]["string"], "C3")
self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["fail"]["string"], "C6")
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["field"], "y")
self.assertAlmostEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["value"], 2.00, places=2)
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["pass"]["string"], "C0")
self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["fail"]["string"], "C0")
engine, = PFAEngine.fromJson(doc)
self.assertEqual(engine.action({"x": 2.0, "y": 3.0}), "C3")
self.assertEqual(engine.action({"x": 2.0, "y": 8.0}), "C6")
self.assertEqual(engine.action({"x": 7.0, "y": 1.0}), "C0")
self.assertEqual(engine.action({"x": 7.0, "y": 5.0}), "C0")
doc = tree.pfaDocument(
{"type": "record", "name": "Datum", "fields": [{"name": "x", "type": "double"}, {"name": "y", "type": "double"}]},
"TreeNode",
nodeScores=True, datasetSize=True, predictandDistribution=True, predictandUnique=True, entropy=True, gain=True)
# look(doc, maxDepth=8)
engine, = PFAEngine.fromJson(doc)
def fromPFA(cls, pfaDoc, ext):
from titus.genpy import PFAEngine
if ext in (".yml", ".yaml"):
engine = PFAEngine.fromYaml(pfaDoc)[0]
else:
engine = PFAEngine.fromJson(pfaDoc)[0]
return cls(engine, "PFA")
def test_main_classification(mock_parameters, mock_save_results,
mock_get_results, mock_fetch_data, method, name):
# create mock objects from database
mock_parameters.return_value = {'type': method}
mock_fetch_data.return_value = fx.inputs_classification(
include_categorical=True)
mock_get_results.return_value = None
main(job_id=None, generate_pfa=True)
pfa = mock_save_results.call_args[0][0]
pfa_dict = json.loads(pfa)
# NOTE: this does not work due to bug in jsonpickle
# deserialize model
# estimator = deserialize_sklearn_estimator(pfa_dict['metadata']['estimator'])
# assert estimator.__class__.__name__ == name
# make some prediction with PFA
from titus.genpy import PFAEngine
engine, = PFAEngine.fromJson(pfa_dict)
engine.action({
'stress_before_test1': 10.,
'iq': 10.,
'agegroup': '50-59y'
})
def testTop5List(self):
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: array(string)
cells:
mostPlanets(array(Star)) = []
action:
// update the list of stars, keeping only the 5 with the most planets
var currentList =
mostPlanets to fcn(old: array(Star) -> array(Star))
stat.sample.topN(input, old, 5, u.morePlanets);
// map this top 5 list of stars to their names
a.map(currentList, fcn(x: Star -> string) x.name)
fcns:
// our comparison function
morePlanets = fcn(x: Star, y: Star -> boolean) a.len(x.planets) < a.len(y.planets)
'''.replace("<<INPUT>>", open("test/prettypfa/exoplanetsSchema.ppfa").read()), check=False, lineNumbers=False)
engine, = PFAEngine.fromJson(pfaDocument)
self.assertEqual(self.runEngine(engine), ["KOI-351", "HD 40307", "GJ 667C", "Kepler-11", "HD 10180"])
def testTree(self):
engine, = PFAEngine.fromJson(open("test/hipparcos_numerical_10.pfa"))
data = []
for line in open("test/hipparcos_numerical.csv"):
ra, dec, dist, mag, absmag, x, y, z, vx, vy, vz, spectrum = line.split(
",")
data.append({
"ra": float(ra),
"dec": float(dec),
"dist": float(dist),
"mag": float(mag),
"absmag": float(absmag),
"x": float(x),
"y": float(y),
"z": float(z),
"vx": float(vx),
"vy": float(vy),
"vz": float(vz)
})
i = 0
startTime = time.time()
for datum in data:
engine.action(datum)
i += 1
if i % 5000 == 0:
print "{0}, {1}".format(time.time() - startTime, i)
def testSimpleKMeansWithStrings(self):
# define the workflow, leaving clusters as an empty array for now
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: string
cells:
clusters(array(record(id: string, center: array(double)))) = []
action:
// ifnotnull runs the first block if all four expressions are not null
// input.mag has type union(double, null) while mag has type double, etc.
ifnotnull(mag: input.mag,
dist: input.dist,
mass: input.mass,
radius: input.radius)
model.cluster.closest(new(array(double), mag, dist, mass, radius),
clusters,
metric.simpleEuclidean)["id"]
else
"MISSING"
'''.replace("<<INPUT>>",
open("test/prettypfa/exoplanetsSchema.ppfa").read()))
# fill in the clusters with the k-means result
if self.kmeansResult is None: self.doKmeans()
pfaDocument["cells"]["clusters"]["init"] = self.kmeansResult.pfaValue(
self.clusterNames)
# build a scoring engine and test it
engine, = PFAEngine.fromJson(pfaDocument)
self.runEngine(engine)
def testTop5List(self):
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: array(string)
cells:
mostPlanets(array(Star)) = []
action:
// update the list of stars, keeping only the 5 with the most planets
var currentList =
mostPlanets to fcn(old: array(Star) -> array(Star))
stat.sample.topN(input, old, 5, u.morePlanets);
// map this top 5 list of stars to their names
a.map(currentList, fcn(x: Star -> string) x.name)
fcns:
// our comparison function
morePlanets = fcn(x: Star, y: Star -> boolean) a.len(x.planets) < a.len(y.planets)
'''.replace("<<INPUT>>", TestClustering.recordSchema),
check=False,
lineNumbers=False)
engine, = PFAEngine.fromJson(pfaDocument)
self.assertEqual(
self.runEngine(engine),
["KOI-351", "HD 40307", "GJ 667C", "Kepler-11", "HD 10180"])
def testSimpleKMeansWithStrings(self):
# define the workflow, leaving clusters as an empty array for now
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: string
cells:
clusters(array(record(id: string, center: array(double)))) = []
action:
// ifnotnull runs the first block if all four expressions are not null
// input.mag has type union(double, null) while mag has type double, etc.
ifnotnull(mag: input.mag,
dist: input.dist,
mass: input.mass,
radius: input.radius)
model.cluster.closest(new(array(double), mag, dist, mass, radius),
clusters,
metric.simpleEuclidean)["id"]
else
"MISSING"
'''.replace("<<INPUT>>", open("test/prettypfa/exoplanetsSchema.ppfa").read()))
# fill in the clusters with the k-means result
if self.kmeansResult is None: self.doKmeans()
pfaDocument["cells"]["clusters"]["init"] = self.kmeansResult.pfaValue(self.clusterNames)
# build a scoring engine and test it
engine, = PFAEngine.fromJson(pfaDocument)
self.runEngine(engine)
def testSimpleKMeansWithEnums(self):
# same as the above using enums rather than strings and compacted a bit
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: enum([cluster0, cluster1, cluster2, cluster3, cluster4, MISSING], ClusterId)
cells:
clusters(array(record(id: ClusterId, center: array(double)))) = []
action:
ifnotnull(mag: input.mag, dist: input.dist, mass: input.mass, radius: input.radius)
model.cluster.closest(new(array(double), mag, dist, mass, radius),
clusters,
metric.simpleEuclidean)["id"]
else
ClusterId@MISSING
'''.replace("<<INPUT>>",
open("test/prettypfa/exoplanetsSchema.ppfa").read()))
if self.kmeansResult is None: self.doKmeans()
pfaDocument["cells"]["clusters"]["init"] = self.kmeansResult.pfaValue(
self.clusterNames)
engine, = PFAEngine.fromJson(pfaDocument)
self.runEngine(engine)
def testPopulationOfClosestCluster(self):
# now that the ifnotnull clause has become three lines long, notice that it needs to be
# surrounded by curly brackets and expressions must be separated by semicolons
# (the last semicolon is optional: they're delimiters, not line terminators)
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: union(int, null)
cells:
clusters(array(record(id: string, center: array(double), population: int))) = []
action:
ifnotnull(mag: input.mag, dist: input.dist, mass: input.mass, radius: input.radius)
model.cluster.closest(new(array(double), mag, dist, mass, radius),
clusters,
metric.simpleEuclidean)["population"]
else
null
'''.replace("<<INPUT>>",
open("test/prettypfa/exoplanetsSchema.ppfa").read()))
if self.kmeansResult is None: self.doKmeans()
pfaDocument["cells"]["clusters"]["init"] = self.kmeansResult.pfaValue(
self.clusterNames, populations=True)
engine, = PFAEngine.fromJson(pfaDocument)
self.runEngine(engine)
def testDistanceToClosestCluster(self):
# now that the ifnotnull clause has become three lines long, notice that it needs to be
# surrounded by curly brackets and expressions must be separated by semicolons
# (the last semicolon is optional: they're delimiters, not line terminators)
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: union(double, null)
cells:
clusters(array(record(id: string, center: array(double)))) = []
action:
ifnotnull(mag: input.mag, dist: input.dist, mass: input.mass, radius: input.radius) {
var datum = new(array(double), mag, dist, mass, radius);
var closestCluster = model.cluster.closest(datum, clusters, metric.simpleEuclidean);
metric.simpleEuclidean(datum, closestCluster["center"])
}
else
null
'''.replace("<<INPUT>>", TestClustering.recordSchema))
if self.kmeansResult is None: self.doKmeans()
pfaDocument["cells"]["clusters"]["init"] = self.kmeansResult.pfaValue(
self.clusterNames)
engine, = PFAEngine.fromJson(pfaDocument)
self.runEngine(engine)
def test_aggregate_knn(mock_save_results, mock_get_results, mock_fetch_data):
# get one PFA
mock_fetch_data.return_value = fx.inputs_regression(include_integer=True)
mock_get_results.return_value = None
compute()
pfa = mock_save_results.call_args[0][0]
def mock_results(job_id):
if job_id == '1':
return mock.MagicMock(data=pfa, error='')
elif job_id == '2':
return mock.MagicMock(data=pfa, error='')
mock_get_results.side_effect = mock_results
aggregate_knn(['1', '2'])
pfa_combined = mock_save_results.call_args[0][0]
pfa_dict = json.loads(pfa_combined)
assert len(pfa_dict['cells']['codebook']['init']) == 2 * len(
json.loads(pfa)['cells']['codebook']['init'])
# make some prediction with PFA
from titus.genpy import PFAEngine
engine, = PFAEngine.fromJson(pfa_dict)
engine.action({
'stress_before_test1': 10.,
'iq': 10.,
'subjectageyears': 70
})
def test_aggregate_kmeans(mock_save_results,
mock_load_intermediate_json_results,
mock_fetch_data):
mock_fetch_data.return_value = fx.inputs_regression(
include_categorical=True)
mock_load_intermediate_json_results.return_value = [
intermediate_data_1(), intermediate_data_2()
]
aggregate_kmeans([1, 2])
pfa_dict = json.loads(mock_save_results.call_args[0][0])
np.testing.assert_allclose(
json.loads(pfa_dict['metadata']['centroids']),
np.array([[-0.12348661147125002, 0.20922071836500003, 0.0, 1.0],
[-0.1852486658437501, 0.09447887226000021, 0.5, 0.0]]), 1e-5)
# make some prediction with PFA
from titus.genpy import PFAEngine
engine, = PFAEngine.fromJson(pfa_dict)
ret = engine.action({
'stress_before_test1': 10.,
'iq': 10.,
'agegroup': '-50y'
})
assert ret == 1
def testKMeansTransform(self):
random.seed(12345)
numpy.seterr(divide="ignore", invalid="ignore")
dataset = numpy.empty((100000, 3), dtype=numpy.dtype(float))
for i, (x, y, z) in enumerate(
TestProducerKMeans.data([2, 1, 1], [3, 2, 5], [8, 2, 7],
[5, 8, 5], [1, 1, 9])):
if i >= dataset.shape[0]:
break
dataset[i, :] = [x * 10.0, y * 20.0, z * 30.0]
trans = Transformation("x/10.0", "y/20.0", "z/30.0")
kmeans = KMeans(5, trans.transform(dataset, ["x", "y", "z"]))
kmeans.optimize(whileall(moving(), maxIterations(1000)))
centers = kmeans.centers()
self.assertArrayAlmostEqual(centers[0], [1.01, 1.00, 9.01], places=1)
self.assertArrayAlmostEqual(centers[1], [2.00, 1.01, 1.00], places=1)
self.assertArrayAlmostEqual(centers[2], [3.01, 2.01, 5.00], places=1)
self.assertArrayAlmostEqual(centers[3], [4.99, 8.00, 4.99], places=1)
self.assertArrayAlmostEqual(centers[4], [8.02, 2.00, 7.01], places=1)
doc = kmeans.pfaDocument("Cluster",
["one", "two", "three", "four", "five"],
preprocess=trans.new(AvroArray(AvroDouble()),
x="input[0]",
y="input[1]",
z="input[2]"))
# look(doc, maxDepth=10)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][0]["center"], [1.01, 1.00, 9.01],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][1]["center"], [2.00, 1.01, 1.00],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][2]["center"], [3.01, 2.01, 5.00],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][3]["center"], [4.99, 8.00, 4.99],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][4]["center"], [8.02, 2.00, 7.01],
places=1)
engine, = PFAEngine.fromJson(doc)
self.assertEqual(engine.action([1.01 * 10, 1.00 * 20, 9.01 * 30]),
"one")
self.assertEqual(engine.action([2.00 * 10, 1.01 * 20, 1.00 * 30]),
"two")
self.assertEqual(engine.action([3.01 * 10, 2.01 * 20, 5.00 * 30]),
"three")
self.assertEqual(engine.action([4.99 * 10, 8.00 * 20, 4.99 * 30]),
"four")
self.assertEqual(engine.action([8.02 * 10, 2.00 * 20, 7.01 * 30]),
"five")
def get_engine(json_string):
"""Creates a PFA engine based on the json_string provided as constructor class. If an
engine was already created, this method does nothing"""
try:
# pylint: disable=unbalanced-tuple-unpacking
engine, = PFAEngine.fromJson(json.loads(json_string))
except ValueError as ex:
# JSON validation
logging.error(
"The file provided does not contain a valid JSON document: " +
str(ex))
sys.exit(1)
except PFASyntaxException as ex:
# Syntax validation
logging.error(
"The file provided does not contain a valid PFA compliant document: "
+ str(ex))
sys.exit(1)
except PFASemanticException as ex:
# PFA semantic check
logging.error(
"The file provided contains inconsistent PFA semantics: " +
str(ex))
sys.exit(1)
except PFAInitializationException as ex:
# Scoring engine check
logging.error(
"It wasn't possible to build a valid scoring engine from the PFA document: "
+ str(ex))
sys.exit(1)
except Exception as ex:
# Other exceptions
logging.error("An unknown exception occurred: " + str(ex))
sys.exit(1)
# Check that the PFA file uses the "map" method. Other methods are not supported
# (because irrelevant) by the MIP
if not engine.config.method == "map":
logging.error(
"The PFA method you used is not supported. Please use the PFA 'map' method"
)
sys.exit(1)
# Check that the PFA file uses a "record" type as input
if not isinstance(engine.config.input, AvroRecord):
logging.error("The PFA document must take a record as input parameter. " \
"Each field of the record must describe a variable")
sys.exit(1)
# Check that the PFA file has a least one input field
if not engine.config.input.fields:
logging.error(
"The PFA document must describe an input record with at least one field"
)
sys.exit(1)
return engine
def __init__(self, config):
# Opening JSON file
f = open('./iris-pfa.json', )
# returns JSON object as a dictionary
pfaDocument = json.load(f)
self.engine, = PFAEngine.fromJson(pfaDocument)
# Closing file
f.close()
def on_put_model(self,request,response):
url = request.path
splitted = url.split("/")
parentpath = os.path.join('/',*splitted[0:-1])
modelname = splitted[-1]
if self.path_exists(url):
path,file_id,parent_id,content,engine = self.cache.getFile(url)
# overwrite content, restart engine
newcontent = request.stream.read(request.content_length or 0)
newengine, = PFAEngine.fromJson(json.loads(newcontent))
newengine.begin()
self.queries.updateFile(file_id = file_id,content = newcontent)
# update cache
self.cache.path2file[url] = (url,file_id,parent_id,newcontent,newengine)
elif self.path_exists(parentpath) and self.is_path_dir(parentpath): # create model in dir, if parent directory exists, start engine
path,file_id,parent_id,content,engine = self.cache.getFile(parentpath)
content = request.stream.read(request.content_length or 0)
engine, = PFAEngine.fromJson(json.loads(content))
engine.begin()
# read the file-rights of the directory
read_owner,write_owner,execute_owner,read_group,write_group,execute_group,read_other,write_other,execute_other = self.cache.getFileRights(file_id)
owner_id, group_id = self.cache.getFileOwners(file_id)
new_file_id = self.queries.insertFile(
content = content,
name = modelname,
parent_id = file_id,
owner_id = owner_id,
group_id = group_id,
read_owner = read_owner,
write_owner = write_owner,
execute_owner = execute_owner,
read_group = read_group,
write_group = write_group,
execute_group = execute_group,
read_other = read_other,
write_other = write_other,
execute_other = execute_other
)
# insert new file in cache
self.cache.path2file[url] = (url,new_file_id,parent_id,content,engine)
self.cache.filesMetaData[new_file_id] = (owner_id, group_id, read_owner,read_group,read_other,write_owner,write_group,write_other,execute_owner,execute_group,execute_other)
response.status = falcon.HTTP_200
else:
response.body = "%s not found" % url
response.status = falcon.HTTP_404
def get_engine(self):
"""Creates a PFA engine based on the json_string provided as constructor class. If an
engine was already created, this method does nothing"""
if not self.engine:
# pylint: disable=unbalanced-tuple-unpacking
engine, = PFAEngine.fromJson(json.loads(self.json_string))
self.engine = engine
return self.engine
def score_model(partition):
# Create PFA engine
from titus.genpy import PFAEngine
engine, = PFAEngine.fromJson(pfa_model.value)
# Score Partition data row-by-row
score_results = list()
for row in partition:
score_results.append([engine.action(row.asDict())])
return score_results
def __load_cache(self):
fileMetaData = self.cursor.execute("select file_id, owner_id, group_id, read_owner,write_owner,execute_owner,read_group,write_group,execute_group,read_other,write_other,execute_other from s_file;").fetchall()
self.filesMetaData = dict([ (fileMetaData[i][0],fileMetaData[i][1:]) for i in range(len(fileMetaData))] )
files = self.cursor.execute("select gr.path,f.file_id,f.parent_id, f.content from s_file f left join global_rights gr on f.file_id = gr.file_id where gr.uid = 1").fetchall()
self.path2file = {}
for file in files:
path,file_id,parent_id,content = file
if not content is None:
content = str(content)
engine, = PFAEngine.fromJson(json.loads(content))
engine.begin()
self.path2file[path] = (path,file_id,parent_id,content, engine)
else:
self.path2file[path] = (path,file_id,parent_id,None,None)
def testKMeans(self):
random.seed(12345)
numpy.seterr(divide="ignore", invalid="ignore")
dataset = numpy.empty((100000, 3), dtype=numpy.dtype(float))
for i, x in enumerate(
TestProducerKMeans.data([2, 1, 1], [3, 2, 5], [8, 2, 7],
[5, 8, 5], [1, 1, 9])):
if i >= dataset.shape[0]:
break
dataset[i, :] = x
kmeans = KMeans(5, dataset)
kmeans.optimize(whileall(moving(), maxIterations(1000)))
centers = kmeans.centers()
self.assertArrayAlmostEqual(centers[0], [1.01, 1.00, 9.01], places=1)
self.assertArrayAlmostEqual(centers[1], [2.00, 1.01, 1.00], places=1)
self.assertArrayAlmostEqual(centers[2], [3.01, 2.01, 5.00], places=1)
self.assertArrayAlmostEqual(centers[3], [4.99, 8.00, 4.99], places=1)
self.assertArrayAlmostEqual(centers[4], [8.02, 2.00, 7.01], places=1)
doc = kmeans.pfaDocument("Cluster",
["one", "two", "three", "four", "five"])
# look(doc, maxDepth=8)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][0]["center"], [1.01, 1.00, 9.01],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][1]["center"], [2.00, 1.01, 1.00],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][2]["center"], [3.01, 2.01, 5.00],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][3]["center"], [4.99, 8.00, 4.99],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][4]["center"], [8.02, 2.00, 7.01],
places=1)
engine, = PFAEngine.fromJson(doc)
self.assertEqual(engine.action([1.01, 1.00, 9.01]), "one")
self.assertEqual(engine.action([2.00, 1.01, 1.00]), "two")
self.assertEqual(engine.action([3.01, 2.01, 5.00]), "three")
self.assertEqual(engine.action([4.99, 8.00, 4.99]), "four")
self.assertEqual(engine.action([8.02, 2.00, 7.01]), "five")
def test_compute(mock_get_param, mock_save_results, mock_fetch_data):
# create mock objects from database
mock_get_param.return_value = 2
mock_fetch_data.return_value = fx.inputs_regression(
include_categorical=True)
compute()
pfa = mock_save_results.call_args[0][0]
pfa_dict = json.loads(pfa)
# make some prediction with PFA
from titus.genpy import PFAEngine
engine, = PFAEngine.fromJson(pfa_dict)
engine.action({'stress_before_test1': 10., 'iq': 10., 'agegroup': '-50y'})
def _predict_pfa(X, types, pfa):
engine, = PFAEngine.fromJson(pfa)
columns = [c for c, _ in types]
pfa_pred = []
for x in X:
p = {}
for col, e, (_, typ) in zip(columns, x, types):
if typ == 'integer':
p[col] = int(e)
else:
p[col] = e
pfa_pred.append(engine.action(p))
return np.array(pfa_pred)
def testTree(self):
engine, = PFAEngine.fromJson(open("test/hipparcos_numerical_10.pfa"))
data = []
for line in open("test/hipparcos_numerical.csv"):
ra, dec, dist, mag, absmag, x, y, z, vx, vy, vz, spectrum = line.split(",")
data.append({"ra": float(ra), "dec": float(dec), "dist": float(dist), "mag": float(mag), "absmag": float(absmag), "x": float(x), "y": float(y), "z": float(z), "vx": float(vx), "vy": float(vy), "vz": float(vz)})
i = 0
startTime = time.time()
for datum in data:
engine.action(datum)
i += 1
if i % 5000 == 0:
print "{0}, {1}".format(time.time() - startTime, i)
def on_post_dir(self,request,response):
url = request.path
try:
name = request.get_header("filename")
username = request.get_header("username")
if not name.endswith(".pfa"):
raise Exception("not supported file: %s" % name)
path,file_id,parent_id,content,engine = self.cache.getFile(url)
# post model to directory if it does not exist
if not self.path_exists(os.path.join(url,name)):
content = request.stream.read(request.content_length or 0)
engine, = PFAEngine.fromJson(json.loads(content))
engine.begin()
# read the file-rights of the directory
read_owner,write_owner,execute_owner,read_group,write_group,execute_group,read_other,write_other,execute_other = self.cache.getFileRights(file_id)
owner_id, group_id = self.cache.getFileOwners(file_id)
new_file_id = self.queries.insertFile(
content = content,
name = name,
parent_id = file_id,
owner_id = owner_id,
group_id = group_id,
read_owner = read_owner,
write_owner = write_owner,
execute_owner = execute_owner,
read_group = read_group,
write_group = write_group,
execute_group = execute_group,
read_other = read_other,
write_other = write_other,
execute_other = execute_other
)
# insert new file in cache
newfilepath = os.path.join(url,name)
self.cache.path2file[newfilepath] = (newfilepath,new_file_id,parent_id,content,engine)
self.cache.filesMetaData[new_file_id] = (owner_id, group_id, read_owner,read_group,read_other,write_owner,write_group,write_other,execute_owner,execute_group,execute_other)
response.status = falcon.HTTP_200
else:
modelpath = os.path.join(url,name)
response.body = "%s already exists. try PUT %s" % (modelpath,modelpath)
response.status = falcon.HTTP_409 # Conflict, model already exists
except Exception, e:
response.body = "%s an error occured, %s" % (url,e)
response.status = falcon.HTTP_505
def test_compute_regression(mock_save_results, mock_get_results,
mock_fetch_data):
# create mock objects from database
mock_fetch_data.return_value = fx.inputs_regression(include_integer=True)
mock_get_results.return_value = None
compute()
pfa = mock_save_results.call_args[0][0]
pfa_dict = json.loads(pfa)
# make some prediction with PFA
from titus.genpy import PFAEngine
engine, = PFAEngine.fromJson(pfa_dict)
engine.action({
'stress_before_test1': 10.,
'iq': 10.,
'subjectageyears': 70
})
def test_main_distributed(mock_parameters, mock_save_results, mock_get_results, mock_fetch_data, method, name):
mock_parameters.return_value = {'type': method}
mock_fetch_data.return_value = fx.inputs_regression()
mock_get_results.return_value = None
# run intermediate job
main(job_id=None, generate_pfa=False)
mock_get_results.return_value = mock.MagicMock(data=mock_save_results.call_args[0][0])
# generate PFA
main(job_id='1', generate_pfa=True)
pfa = mock_save_results.call_args_list[1][0][0]
pfa_dict = json.loads(pfa)
# make some prediction with PFA
from titus.genpy import PFAEngine
engine, = PFAEngine.fromJson(pfa_dict)
engine.action({'stress_before_test1': 10., 'iq': 10., 'agegroup': '-50y'})
def testKMeansTransform(self):
random.seed(12345)
numpy.seterr(divide="ignore", invalid="ignore")
dataset = numpy.empty((100000, 3), dtype=numpy.dtype(float))
for i, (x, y, z) in enumerate(TestProducerKMeans.data([1, 1, 1], [3, 2, 5], [8, 2, 7], [5, 8, 5], [1, 1, 9])):
if i >= dataset.shape[0]:
break
dataset[i,:] = [x * 10.0, y * 20.0, z * 30.0]
trans = Transformation("x/10.0", "y/20.0", "z/30.0")
kmeans = KMeans(5, trans.transform(dataset, ["x", "y", "z"]))
kmeans.optimize(whileall(moving(), maxIterations(1000)))
centers = kmeans.centers()
self.assertArrayAlmostEqual(centers[0], [1.00, 1.01, 1.00], places=1)
self.assertArrayAlmostEqual(centers[1], [1.01, 1.00, 9.01], places=1)
self.assertArrayAlmostEqual(centers[2], [3.01, 2.01, 5.00], places=1)
self.assertArrayAlmostEqual(centers[3], [4.99, 8.00, 4.99], places=1)
self.assertArrayAlmostEqual(centers[4], [8.02, 2.00, 7.01], places=1)
doc = kmeans.pfaDocument("Cluster",
["one", "two", "three", "four", "five"],
preprocess=trans.new(AvroArray(AvroDouble()),
x="input[0]", y="input[1]", z="input[2]"))
# look(doc, maxDepth=10)
self.assertArrayAlmostEqual(doc["cells"]["clusters"]["init"][0]["center"], [1.00, 1.01, 1.00], places=2)
self.assertArrayAlmostEqual(doc["cells"]["clusters"]["init"][1]["center"], [1.01, 1.00, 9.01], places=2)
self.assertArrayAlmostEqual(doc["cells"]["clusters"]["init"][2]["center"], [3.01, 2.01, 5.00], places=2)
self.assertArrayAlmostEqual(doc["cells"]["clusters"]["init"][3]["center"], [4.99, 8.00, 4.99], places=2)
self.assertArrayAlmostEqual(doc["cells"]["clusters"]["init"][4]["center"], [8.02, 2.00, 7.01], places=2)
engine, = PFAEngine.fromJson(doc)
self.assertEqual(engine.action([1.00 * 10, 1.01 * 20, 1.00 * 30]), "one")
self.assertEqual(engine.action([1.01 * 10, 1.00 * 20, 9.01 * 30]), "two")
self.assertEqual(engine.action([3.01 * 10, 2.01 * 20, 5.00 * 30]), "three")
self.assertEqual(engine.action([4.99 * 10, 8.00 * 20, 4.99 * 30]), "four")
self.assertEqual(engine.action([8.02 * 10, 2.00 * 20, 7.01 * 30]), "five")
def testSimpleKMeansEmitExample(self):
# the emit method allows us to ignore the "else" clause in ifnotnull
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: string
method: emit
cells:
clusters(array(record(id: string, center: array(double)))) = []
action:
ifnotnull(mag: input.mag, dist: input.dist, mass: input.mass, radius: input.radius)
emit(model.cluster.closest(new(array(double), mag, dist, mass, radius),
clusters,
metric.simpleEuclidean)["id"])
'''.replace("<<INPUT>>", open("test/prettypfa/exoplanetsSchema.ppfa").read()))
if self.kmeansResult is None: self.doKmeans()
pfaDocument["cells"]["clusters"]["init"] = self.kmeansResult.pfaValue(self.clusterNames)
engine, = PFAEngine.fromJson(pfaDocument)
self.runEngine(engine)
def testSimpleKMeansWithEnums(self):
# same as the above using enums rather than strings and compacted a bit
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: enum([cluster0, cluster1, cluster2, cluster3, cluster4, MISSING], ClusterId)
cells:
clusters(array(record(id: ClusterId, center: array(double)))) = []
action:
ifnotnull(mag: input.mag, dist: input.dist, mass: input.mass, radius: input.radius)
model.cluster.closest(new(array(double), mag, dist, mass, radius),
clusters,
metric.simpleEuclidean)["id"]
else
ClusterId@MISSING
'''.replace("<<INPUT>>", open("test/prettypfa/exoplanetsSchema.ppfa").read()))
if self.kmeansResult is None: self.doKmeans()
pfaDocument["cells"]["clusters"]["init"] = self.kmeansResult.pfaValue(self.clusterNames)
engine, = PFAEngine.fromJson(pfaDocument)
self.runEngine(engine)
def testPopulationOfClosestCluster(self):
# now that the ifnotnull clause has become three lines long, notice that it needs to be
# surrounded by curly brackets and expressions must be separated by semicolons
# (the last semicolon is optional: they're delimiters, not line terminators)
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: union(int, null)
cells:
clusters(array(record(id: string, center: array(double), population: int))) = []
action:
ifnotnull(mag: input.mag, dist: input.dist, mass: input.mass, radius: input.radius)
model.cluster.closest(new(array(double), mag, dist, mass, radius),
clusters,
metric.simpleEuclidean)["population"]
else
null
'''.replace("<<INPUT>>", open("test/prettypfa/exoplanetsSchema.ppfa").read()))
if self.kmeansResult is None: self.doKmeans()
pfaDocument["cells"]["clusters"]["init"] = self.kmeansResult.pfaValue(self.clusterNames, populations=True)
engine, = PFAEngine.fromJson(pfaDocument)
self.runEngine(engine)
def testSimpleKMeansEmitExample(self):
# the emit method allows us to ignore the "else" clause in ifnotnull
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: string
method: emit
cells:
clusters(array(record(id: string, center: array(double)))) = []
action:
ifnotnull(mag: input.mag, dist: input.dist, mass: input.mass, radius: input.radius)
emit(model.cluster.closest(new(array(double), mag, dist, mass, radius),
clusters,
metric.simpleEuclidean)["id"])
'''.replace("<<INPUT>>", TestClustering.recordSchema))
if self.kmeansResult is None: self.doKmeans()
pfaDocument["cells"]["clusters"]["init"] = self.kmeansResult.pfaValue(
self.clusterNames)
engine, = PFAEngine.fromJson(pfaDocument)
self.runEngine(engine)
def testHistogram2d(self):
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: Histogram
cells:
histogram(record(Histogram,
xnumbins: int,
xlow: double,
xhigh: double,
ynumbins: int,
ylow: double,
yhigh: double,
values: array(array(double)))) = {
xnumbins: 10, xlow: 0.0, xhigh: 3.0,
ynumbins: 10, ylow: 0.0, yhigh: 3.0,
values: [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]}
method: emit
action:
ifnotnull(mass: input.mass, radius: input.radius)
emit(histogram to fcn(old: Histogram -> Histogram)
stat.sample.fillHistogram2d(mass, radius, 1.0, old))
'''.replace("<<INPUT>>", open("test/prettypfa/exoplanetsSchema.ppfa").read()), check=False, lineNumbers=False)
engine, = PFAEngine.fromJson(pfaDocument)
self.assertEqual(self.runEngine(engine), {"values": [[6.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.0, 33.0, 3.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 8.0, 118.0, 28.0, 6.0, 0.0, 1.0, 1.0, 0.0, 0.0], [0.0, 0.0, 33.0, 184.0, 72.0, 25.0, 8.0, 4.0, 0.0, 1.0], [0.0, 0.0, 1.0, 12.0, 45.0, 34.0, 20.0, 3.0, 4.0, 1.0], [0.0, 0.0, 0.0, 1.0, 1.0, 4.0, 4.0, 4.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "xhigh": 3.0, "yhigh": 3.0, "ynumbins": 10, "xnumbins": 10, "ylow": 0.0, "xlow": 0.0})
def testKMeans(self):
random.seed(12345)
numpy.seterr(divide="ignore", invalid="ignore")
dataset = numpy.empty((100000, 3), dtype=numpy.dtype(float))
for i, x in enumerate(TestProducerKMeans.data([1, 1, 1], [3, 2, 5], [8, 2, 7], [5, 8, 5], [1, 1, 9])):
if i >= dataset.shape[0]:
break
dataset[i,:] = x
kmeans = KMeans(5, dataset)
kmeans.optimize(whileall(moving(), maxIterations(1000)))
centers = kmeans.centers()
self.assertArrayAlmostEqual(centers[0], [1.00, 1.01, 1.00], places=2)
self.assertArrayAlmostEqual(centers[1], [1.01, 1.00, 9.01], places=2)
self.assertArrayAlmostEqual(centers[2], [3.01, 2.01, 5.00], places=2)
self.assertArrayAlmostEqual(centers[3], [4.99, 8.00, 4.99], places=2)
self.assertArrayAlmostEqual(centers[4], [8.02, 2.00, 7.01], places=2)
doc = kmeans.pfaDocument("Cluster", ["one", "two", "three", "four", "five"])
# look(doc, maxDepth=8)
self.assertArrayAlmostEqual(doc["cells"]["clusters"]["init"][0]["center"], [1.00, 1.01, 1.00], places=2)
self.assertArrayAlmostEqual(doc["cells"]["clusters"]["init"][1]["center"], [1.01, 1.00, 9.01], places=2)
self.assertArrayAlmostEqual(doc["cells"]["clusters"]["init"][2]["center"], [3.01, 2.01, 5.00], places=2)
self.assertArrayAlmostEqual(doc["cells"]["clusters"]["init"][3]["center"], [4.99, 8.00, 4.99], places=2)
self.assertArrayAlmostEqual(doc["cells"]["clusters"]["init"][4]["center"], [8.02, 2.00, 7.01], places=2)
engine, = PFAEngine.fromJson(doc)
self.assertEqual(engine.action([1.00, 1.01, 1.00]), "one")
self.assertEqual(engine.action([1.01, 1.00, 9.01]), "two")
self.assertEqual(engine.action([3.01, 2.01, 5.00]), "three")
self.assertEqual(engine.action([4.99, 8.00, 4.99]), "four")
self.assertEqual(engine.action([8.02, 2.00, 7.01]), "five")
def testHistogram2d(self):
pfaDocument = titus.prettypfa.jsonNode('''
input: <<INPUT>>
output: Histogram
cells:
histogram(record(Histogram,
xnumbins: int,
xlow: double,
xhigh: double,
ynumbins: int,
ylow: double,
yhigh: double,
values: array(array(double)))) = {
xnumbins: 10, xlow: 0.0, xhigh: 3.0,
ynumbins: 10, ylow: 0.0, yhigh: 3.0,
values: [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]}
method: emit
action:
ifnotnull(mass: input.mass, radius: input.radius)
emit(histogram to fcn(old: Histogram -> Histogram)
stat.sample.fillHistogram2d(mass, radius, 1.0, old))
'''.replace("<<INPUT>>", TestClustering.recordSchema),
check=False,
lineNumbers=False)
engine, = PFAEngine.fromJson(pfaDocument)
self.assertEqual(
self.runEngine(engine), {
"values":
[[6.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[3.0, 33.0, 3.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 8.0, 118.0, 28.0, 6.0, 0.0, 1.0, 1.0, 0.0, 0.0],
[0.0, 0.0, 33.0, 184.0, 72.0, 25.0, 8.0, 4.0, 0.0, 1.0],
[0.0, 0.0, 1.0, 12.0, 45.0, 34.0, 20.0, 3.0, 4.0, 1.0],
[0.0, 0.0, 0.0, 1.0, 1.0, 4.0, 4.0, 4.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
"xhigh":
3.0,
"yhigh":
3.0,
"ynumbins":
10,
"xnumbins":
10,
"ylow":
0.0,
"xlow":
0.0
})
inputFile, = sys.argv[1:]
# Failures that I'm giving up on:
#
# prob.dist.binomialQF({"p": 0.99999, "prob": 1e-05, "size": 1}) should be 1, is 0 (rounding in count)
# {"p": 0.9, "prob": 0.1, "size": 1} should be 1, is 0 (same reason)
# prob.dist.hypergeometricPDF \
# prob.dist.hypergeometricCDF } many errors! and the QF has a long or infinite loop
# prob.dist.hypergeometricQF /
# prob.dist.negativeBinomialPDF({"x": 17, "prob": 0.9, "size": 100}) should be 0.00245, is 0.02715
# {"x": 100, "prob": 0.1, "size": 17} should be 0.00245, is 0.00462
# {"x": 100, "prob": 0.5, "size": 100} should be 5.7e42, is 0.02817
# prob.dist.negativeBinomialQF has many errors (though not as many as the hypergeometric)
for counter, example in enumerate(getExamples(open(inputFile))):
engine, = PFAEngine.fromJson(example["engine"])
if example["function"] in ("prob.dist.binomialQF", "prob.dist.hypergeometricPDF", "prob.dist.hypergeometricCDF", "prob.dist.hypergeometricQF", "prob.dist.negativeBinomialPDF", "prob.dist.negativeBinomialQF"):
continue
functionWritten = False
def maybeWriteFunction(functionWritten):
if not functionWritten:
print "%4d %-20s%s" % (counter + 1, example["function"], json.dumps(example["engine"]))
return True
for trial in example["trials"]:
trialWritten = False
try:
result = {"success": convertOut(engine.action(trial["sample"]), engine.outputType.jsonNode(set()), dobase64=True)}
except PFARuntimeException as err:
server_address = sys.argv[1]
pfa_model = sys.argv[2]
kafka_topic_in = sys.argv[3]
kafka_topic_out = sys.argv[4]
print "================"
print "Kafka Scoring"
print "Bootstrap server:\t%s" % server_address
print "PFA model:\t\t%s" % pfa_model
print "Topic consumed:\t\t%s" % kafka_topic_in
print "Topic produced:\t\t%s" % kafka_topic_out
# Create PFA engine
try:
pfa_engine, = PFAEngine.fromJson(json.load(open(pfa_model)))
except Exception:
sys.exit("Failed to create scoring engine")
# Initialize PFA engine
pfa_engine.begin()
# Configure Kafka connection
try:
consumer = KafkaConsumer(kafka_topic_in, bootstrap_servers=server_address)
producer = KafkaProducer(bootstrap_servers=server_address)
except Exception:
sys.exit("Failed to configure Kafka")
count = 0
bad_data = 0
def testNormalized(self):
# for k-means on normalized data, we have to explicitly normalize,
# re-compute the clusters, and put the same transformation into PFA
# get a dataset for the k-means generator
dataset = []
for record in DataFileReader(open("test/prettypfa/exoplanets.avro", "r"), DatumReader()):
mag, dist, mass, radius = record.get("mag"), record.get("dist"), record.get("mass"), record.get("radius")
if mag is not None and dist is not None and mass is not None and radius is not None:
dataset.append([mag, dist, mass, radius])
dataset = numpy.array(dataset)
# compute the normalization (1st to 99th percentile instead of strict min/max)
maglow, maghigh = numpy.percentile(dataset[:,0], [1, 99])
distlow, disthigh = numpy.percentile(dataset[:,1], [1, 99])
masslow, masshigh = numpy.percentile(dataset[:,2], [1, 99])
radiuslow, radiushigh = numpy.percentile(dataset[:,3], [1, 99])
# transform the data
normalized = numpy.empty_like(dataset)
normalized[:,0] = (dataset[:,0] - maglow) / (maghigh - maglow)
normalized[:,1] = (dataset[:,1] - distlow) / (disthigh - distlow)
normalized[:,2] = (dataset[:,2] - masslow) / (masshigh - masslow)
normalized[:,3] = (dataset[:,3] - radiuslow) / (radiushigh - radiuslow)
# set up and run the k-means generator
kmeansResult = KMeans(len(self.clusterNames), normalized)
kmeansResult.optimize(whileall(moving(), maxIterations(1000)))
# put the transformation into PFA by string replacement
# this re.subs will replace French quotes (<< >>) with Python variable values
inputSchema = open("test/prettypfa/exoplanetsSchema.ppfa").read()
namesToSubstitute = locals()
pfaDocument = titus.prettypfa.jsonNode(
re.sub("<<[A-Za-z0-9]+>>",
lambda x: str(namesToSubstitute[x.group().lstrip("<<").rstrip(">>")]),
'''
input: <<inputSchema>>
output: string
cells:
clusters(array(record(id: string, center: array(double)))) = []
action:
ifnotnull(mag: input.mag, dist: input.dist, mass: input.mass, radius: input.radius) {
var normmag = (mag - <<maglow>>) / (<<maghigh>> - <<maglow>>);
var normdist = (dist - <<distlow>>) / (<<disthigh>> - <<distlow>>);
var normmass = (mass - <<masslow>>) / (<<masshigh>> - <<masslow>>);
var normradius = (radius - <<radiuslow>>) / (<<radiushigh>> - <<radiuslow>>);
model.cluster.closest(new(array(double), normmag, normdist, normmass, normradius),
clusters,
metric.simpleEuclidean)["id"]
}
else
"MISSING"
'''))
# now put the clusters in and run the scoring engine
pfaDocument["cells"]["clusters"]["init"] = kmeansResult.pfaValue(self.clusterNames)
engine, = PFAEngine.fromJson(pfaDocument)
self.runEngine(engine)
def post(self, model_name):
pfa_model = db.get_model(model_name)
pfa_engine, = PFAEngine.fromJson(pfa_model)
data_to_score = tornado.escape.json_decode(self.request.body)
db.update_usage_stats(model_name)
self.write(str(pfa_engine.action(data_to_score)))
inputFile, = sys.argv[1:]
# Failures that I'm giving up on:
#
# prob.dist.binomialQF({"p": 0.99999, "prob": 1e-05, "size": 1}) should be 1, is 0 (rounding in count)
# {"p": 0.9, "prob": 0.1, "size": 1} should be 1, is 0 (same reason)
# prob.dist.hypergeometricPDF \
# prob.dist.hypergeometricCDF } many errors! and the QF has a long or infinite loop
# prob.dist.hypergeometricQF /
# prob.dist.negativeBinomialPDF({"x": 17, "prob": 0.9, "size": 100}) should be 0.00245, is 0.02715
# {"x": 100, "prob": 0.1, "size": 17} should be 0.00245, is 0.00462
# {"x": 100, "prob": 0.5, "size": 100} should be 5.7e42, is 0.02817
# prob.dist.negativeBinomialQF has many errors (though not as many as the hypergeometric)
for counter, example in enumerate(getExamples(open(inputFile))):
engine, = PFAEngine.fromJson(example["engine"])
if example["function"] in ("prob.dist.binomialQF",
"prob.dist.hypergeometricPDF",
"prob.dist.hypergeometricCDF",
"prob.dist.hypergeometricQF",
"prob.dist.negativeBinomialPDF",
"prob.dist.negativeBinomialQF"):
continue
functionWritten = False
def maybeWriteFunction(functionWritten):
if not functionWritten:
print "%4d %-20s%s" % (counter + 1, example["function"],
json.dumps(example["engine"]))
