def testInputLabelListProperlyInitialized(self):
settings = {}
settings["apiKey"] = "xxx"
settings["algoSingle"] = "userName/algoName"
settings["inputLabelList"] = "an input"
with pytest.raises(AlgoBenchError):
b = Benchmark(settings)
settings["inputLabelList"] = ["an input", "another input"]
with pytest.raises(AlgoBenchError):
b2 = Benchmark(settings)
settings["inputLabelList"] = [{
"data": "an input",
"label": "true"
}, {
"data": "another input",
"label": "false"
}]
b3 = Benchmark(settings)
assert "inputLabelList" in b3.settings
assert "inputList" not in b3.settings
assert "inputSingle" not in b3.settings
def testCalcBasics(self):
settings = {}
settings["apiKey"] = "xxx"
settings["algoSingle"] = "userName/algoName"
settings["inputSingle"] = "an input"
b = Benchmark(settings)
b.results = [{"result": 5, "label": 5}, {"result": 7, "label": 7},\
{"result": 5, "label": 5}, {"result": 1, "label": 7},\
{"result": 3, "label": 5}, {"result": 7, "label": 7},\
{"result": 5, "label": 5}, {"result": 4, "label": 7},\
{"result": 4, "label": 5}, {"result": 7, "label": 7}]
def mapFunc(res):
return res
b.calcStats(mapFunc)
assert b.stats["FN"]["labels"][5] == 2
assert b.stats["FN"]["labels"][7] == 2
assert b.stats["FP"]["labels"][5] == 0
assert b.stats["FP"]["labels"][7] == 0
assert b.stats["TN"]["labels"][5] == 5
assert b.stats["TN"]["labels"][7] == 5
assert b.stats["TP"]["labels"][5] == 3
assert b.stats["TP"]["labels"][7] == 3
def testHasLabels(self):
settings = {}
settings["apiKey"] = "xxx"
settings["algoSingle"] = "userName/algoName"
settings["inputSingle"] = "an input"
b = Benchmark(settings)
b.results = [{"result": 5, "label": 5}, {"result": 7, "label": 7}]
assert b._Benchmark__hasLabels() == True
def testInputListProperlyInitialized(self):
settings = {}
settings["apiKey"] = "xxx"
settings["algoSingle"] = "userName/algoName"
settings["inputList"] = "an input"
with pytest.raises(AlgoBenchError):
b = Benchmark(settings)
settings["inputList"] = ["an input", "another input"]
b2 = Benchmark(settings)
assert "inputLabelList" in b2.settings
assert "inputList" not in b2.settings
def testCalcAverage(self):
settings = {}
settings["apiKey"] = "xxx"
settings["inputSingle"] = "an input"
settings["algoSingle"] = "userName/algoName"
b = Benchmark(settings)
b.results = [{"response":{"metadata":{"duration":0.24214214}}, "algo":"userName/algoName"},\
{"response":{"metadata":{"duration":0.43523423}}, "algo":"userName/algoName"},\
{"response":{"metadata":{"duration":0.12435256}}, "algo":"userName/algoName"},\
{"response":{"metadata":{"duration":0.35314251}}, "algo":"userName/algoName"},\
{"response":{"metadata":{"duration":0.12451533}}, "algo":"userName/algoName"}]
b._Benchmark__calcAverage()
assert b.average["userName/algoName"] == 0.255877354
def testAlgoListProperlyInitialized(self):
settings = {}
settings["apiKey"] = "xxx"
settings["algoList"] = "userName/algoName"
settings["inputSingle"] = "an input"
with pytest.raises(AlgoBenchError):
b = Benchmark(settings)
settings["algoList"] = [
"userName/algoName/ver1", "userName/algoName/ver2"
]
b2 = Benchmark(settings)
assert "algoList" in b2.settings
def testInputSingleProperlyInitialized(self):
settings = {}
settings["apiKey"] = "xxx"
settings["algoSingle"] = "userName/algoName"
settings["inputSingle"] = "an input"
b = Benchmark(settings)
assert "inputLabelList" in b.settings
assert "inputSingle" not in b.settings
def testNoMaxNumConnections(self):
settings = {}
settings["apiKey"] = "xxx"
settings["inputSingle"] = "an input"
settings["algoSingle"] = "userName/algoName"
settings["numBenchRuns"] = 5
b = Benchmark(settings)
assert b.settings["maxNumConnections"] == 10
def testCalcFScore(self):
settings = {}
settings["apiKey"] = "xxx"
settings["algoSingle"] = "userName/algoName"
settings["inputSingle"] = "an input"
b = Benchmark(settings)
b.results = [{"result": 5, "label": 5}, {"result": 7, "label": 7},\
{"result": 5, "label": 5}, {"result": 1, "label": 7},\
{"result": 3, "label": 5}, {"result": 7, "label": 7},\
{"result": 5, "label": 5}, {"result": 4, "label": 7},\
{"result": 4, "label": 5}, {"result": 7, "label": 7}]
def mapFunc(res):
return res
b.calcStats(mapFunc)
assert b.stats['fScore']['labels'][5] == 0.7499999999999999
assert b.stats['fScore']['labels'][7] == 0.7499999999999999
def testSettingsIsNotDict(self):
settings = "not dict"
with pytest.raises(AlgoBenchError):
b = Benchmark(settings)
def testNoAlgo(self):
settings = {}
settings["apiKey"] = "xxx"
settings["inputSingle"] = "an input"
with pytest.raises(AlgoBenchError):
b = Benchmark(settings)
def testNoInput(self):
settings = {}
settings["apiKey"] = "xxx"
settings["algoSingle"] = "userName/algoName"
with pytest.raises(AlgoBenchError):
b = Benchmark(settings)
def testNoAPIKey(self):
settings = {}
settings["inputSingle"] = "an input"
settings["algoSingle"] = "userName/algoName"
with pytest.raises(AlgoBenchError):
b = Benchmark(settings)
