def test_getPermutations(self):
desc = self.fakeDescription()
exp = ExperimentDescription(desc)
# can get permutation of metaParameters by default
got = exp.getPermutation(0)
expected = desc.copy()
expected['metaParameters'] = {
'alpha': 0.01,
'epsilon': 0.05,
'gamma': 0.9,
}
self.assertDictEqual(
got,
expected,
)
# can get permutation of multiple parameters
got = exp.getPermutation(0, keys=['metaParameters', 'envParameters'])
expected = {
'metaParameters': {
'alpha': 0.01,
'epsilon': 0.05,
'gamma': 0.9,
},
'envParameters': {
'size': 30,
'noise': 0.01,
},
}
self.assertDictEqual(got, expected)
def listResultsPaths(exp: ExperimentDescription,
runs: int = 1,
key: Optional[str] = None):
perms = exp.numPermutations()
tasks = perms * runs
for i in range(tasks):
yield exp.interpolateSavePath(i, key=key)
def test_permutable(self):
desc = self.fakeDescription()
exp = ExperimentDescription(desc)
# permutable defaults to 'metaParameters' being the only permutable key
got = exp.permutable()
self.assertDictEqual(
got,
{'metaParameters': desc['metaParameters']},
)
# can specify a list of parameters to permute over
got = exp.permutable(['metaParameters', 'envParameters'])
self.assertDictEqual(got, desc)
def test_permutations(self):
desc = self.fakeDescription()
exp = ExperimentDescription(desc)
got = exp.numPermutations()
expected = 3
self.assertEqual(got, expected)
got = exp.numPermutations(keys='envParameters')
expected = 2
self.assertEqual(got, expected)
got = exp.numPermutations(keys=['metaParameters', 'envParameters'])
expected = 6
self.assertEqual(got, expected)
def test_interpolateSavePath(self):
desc = {
'metaParameters': {
'optimizer': {
'alpha': [0.1, 0.2],
'beta': [0.99, 0.999],
},
'epsilon': 0.05,
},
}
exp = ExperimentDescription(desc)
key = '{params}'
got = exp.interpolateSavePath(0, key=key)
expected = 'epsilon-0.05_optimizer.alpha-0.1_optimizer.beta-0.99'
self.assertEqual(got, expected)
def test_permutations(self):
desc = self.fakeDescription()
exp = ExperimentDescription(desc)
# compute number of permutations first time
got = exp.numPermutations()
expected = 3
self.assertEqual(got, expected)
# recompute if the keys are different
got = exp.numPermutations(keys='envParameters')
expected = 2
self.assertEqual(got, expected)
# recompute if keys is array
got = exp.numPermutations(keys=['metaParameters', 'envParameters'])
expected = 6
self.assertEqual(got, expected)
def test_listIndices(self):
exp = ExperimentDescription({
'metaParameters': {
'alpha': [0.01, 0.02, 0.04, 0.08, 0.16],
'lambda': [1.0, 0.99, 0.98, 0.96, 0.92],
}
})
expected = list(range(25))
got = list(listIndices(exp))
self.assertListEqual(got, expected)
def __init__(self, path: str, exp: ExperimentDescription, idx: int):
self.path = path
self.exp = exp
self.idx = idx
self.params = exp.getPermutation(idx)[exp._getKeys()[0]]
self._data: Optional[Dict] = None
from PyExpUtils.results.results import Result, whereParametersEqual, getBest, splitOverParameter
from PyExpUtils.results.indices import listIndices
from PyExpUtils.models.ExperimentDescription import ExperimentDescription
import unittest
import numpy as np
exp = ExperimentDescription({
'metaParameters': {
'alpha': [1.0, 0.5, 0.25],
'ratio': [1.0, 2.0, 4.0, 8.0],
'model': {
'name': ['PR', 'ESARSA'],
}
},
})
class TestResults(unittest.TestCase):
def test_Results(self):
results = [Result('fake/path', exp, i) for i in listIndices(exp)]
r = results[0]
self.assertDictEqual(r.params, {
'alpha': 1.0,
'ratio': 1.0,
'model': {
'name': 'PR'
}
})
r = results[1]
self.assertDictEqual(r.params, {
def listIndices(exp: ExperimentDescription, runs: int = 1):
perms = exp.numPermutations()
tasks = perms * runs
return range(tasks)