Beispiel #1
0
    def test_getPermutations(self):
        desc = self.fakeDescription()
        exp = ExperimentDescription(desc)

        # can get permutation of metaParameters by default
        got = exp.getPermutation(0)
        expected = desc.copy()
        expected['metaParameters'] = {
            'alpha': 0.01,
            'epsilon': 0.05,
            'gamma': 0.9,
        }
        self.assertDictEqual(
            got,
            expected,
        )

        # can get permutation of multiple parameters
        got = exp.getPermutation(0, keys=['metaParameters', 'envParameters'])
        expected = {
            'metaParameters': {
                'alpha': 0.01,
                'epsilon': 0.05,
                'gamma': 0.9,
            },
            'envParameters': {
                'size': 30,
                'noise': 0.01,
            },
        }
        self.assertDictEqual(got, expected)
Beispiel #2
0
def listResultsPaths(exp: ExperimentDescription,
                     runs: int = 1,
                     key: Optional[str] = None):
    perms = exp.numPermutations()
    tasks = perms * runs
    for i in range(tasks):
        yield exp.interpolateSavePath(i, key=key)
Beispiel #3
0
    def test_permutable(self):
        desc = self.fakeDescription()
        exp = ExperimentDescription(desc)

        # permutable defaults to 'metaParameters' being the only permutable key
        got = exp.permutable()
        self.assertDictEqual(
            got,
            {'metaParameters': desc['metaParameters']},
        )

        # can specify a list of parameters to permute over
        got = exp.permutable(['metaParameters', 'envParameters'])
        self.assertDictEqual(got, desc)
Beispiel #4
0
    def test_permutations(self):
        desc = self.fakeDescription()
        exp = ExperimentDescription(desc)

        got = exp.numPermutations()
        expected = 3
        self.assertEqual(got, expected)

        got = exp.numPermutations(keys='envParameters')
        expected = 2
        self.assertEqual(got, expected)

        got = exp.numPermutations(keys=['metaParameters', 'envParameters'])
        expected = 6
        self.assertEqual(got, expected)
Beispiel #5
0
    def test_interpolateSavePath(self):
        desc = {
            'metaParameters': {
                'optimizer': {
                    'alpha': [0.1, 0.2],
                    'beta': [0.99, 0.999],
                },
                'epsilon': 0.05,
            },
        }

        exp = ExperimentDescription(desc)
        key = '{params}'

        got = exp.interpolateSavePath(0, key=key)
        expected = 'epsilon-0.05_optimizer.alpha-0.1_optimizer.beta-0.99'
        self.assertEqual(got, expected)
Beispiel #6
0
    def test_permutations(self):
        desc = self.fakeDescription()
        exp = ExperimentDescription(desc)

        # compute number of permutations first time
        got = exp.numPermutations()
        expected = 3
        self.assertEqual(got, expected)

        # recompute if the keys are different
        got = exp.numPermutations(keys='envParameters')
        expected = 2
        self.assertEqual(got, expected)

        # recompute if keys is array
        got = exp.numPermutations(keys=['metaParameters', 'envParameters'])
        expected = 6
        self.assertEqual(got, expected)
Beispiel #7
0
    def test_listIndices(self):
        exp = ExperimentDescription({
            'metaParameters': {
                'alpha': [0.01, 0.02, 0.04, 0.08, 0.16],
                'lambda': [1.0, 0.99, 0.98, 0.96, 0.92],
            }
        })

        expected = list(range(25))
        got = list(listIndices(exp))

        self.assertListEqual(got, expected)
Beispiel #8
0
 def __init__(self, path: str, exp: ExperimentDescription, idx: int):
     self.path = path
     self.exp = exp
     self.idx = idx
     self.params = exp.getPermutation(idx)[exp._getKeys()[0]]
     self._data: Optional[Dict] = None
Beispiel #9
0
from PyExpUtils.results.results import Result, whereParametersEqual, getBest, splitOverParameter
from PyExpUtils.results.indices import listIndices
from PyExpUtils.models.ExperimentDescription import ExperimentDescription
import unittest
import numpy as np

exp = ExperimentDescription({
    'metaParameters': {
        'alpha': [1.0, 0.5, 0.25],
        'ratio': [1.0, 2.0, 4.0, 8.0],
        'model': {
            'name': ['PR', 'ESARSA'],
        }
    },
})


class TestResults(unittest.TestCase):
    def test_Results(self):
        results = [Result('fake/path', exp, i) for i in listIndices(exp)]

        r = results[0]
        self.assertDictEqual(r.params, {
            'alpha': 1.0,
            'ratio': 1.0,
            'model': {
                'name': 'PR'
            }
        })
        r = results[1]
        self.assertDictEqual(r.params, {
Beispiel #10
0
def listIndices(exp: ExperimentDescription, runs: int = 1):
    perms = exp.numPermutations()
    tasks = perms * runs
    return range(tasks)