def __init__(self, values):
self._values = to_list(values)
self._data = reader.read(self._values)
Parser.check_data(spec=self, data=self._data)
headers = Parser.get_headers(spec=self, data=self._data)
matrix = Parser.get_matrix(spec=self, data=self._data)
try:
self._matrix = validator.validate_matrix(matrix)
except ValidationError as e:
raise PolyaxonConfigurationError(e)
try:
self._headers = validator.validate_headers(spec=self, data=headers)
except ValidationError as e:
raise PolyaxonConfigurationError(e)
self._parsed_data = []
self._validated_data = []
self._experiment_specs = []
matrix_declarations = self.matrix_declarations if self.matrix_declarations else [
{}
]
for i, matrix_declaration in enumerate(matrix_declarations):
parsed_data = Parser.parse(self, self._data, matrix_declaration)
self._validated_data.append(
validator.validate(spec=self, data=parsed_data))
self._parsed_data.append(parsed_data)
self._experiment_specs.append(
Specification(experiment=i, values=parsed_data))
def test_for_operator_config(self):
config_dict = {
'len': 5,
'do': 'Value at {{ i }}',
'index': 'i'
}
config = ForConfig.from_dict(config_dict)
assert config.to_dict() == config_dict
expected = ['Value at 0', 'Value at 1', 'Value at 2', 'Value at 3', 'Value at 4']
assert expected == config.parse(ExperimentSpecification, Parser(), {})
config_dict = {
'len': 5,
'do': [{'Conv2D': {'strides': ['{{ i }}', '{{ i }}']}},
{'Pooling2D': {'strides': ['{{ i }}', '{{ i }}']}}],
'index': 'i'
}
config = ForConfig.from_dict(config_dict)
assert config.to_dict() == config_dict
# Lists get flattened
expected = [{'Conv2D': {'strides': [0, 0]}}, {'Pooling2D': {'strides': [0, 0]}},
{'Conv2D': {'strides': [1, 1]}}, {'Pooling2D': {'strides': [1, 1]}},
{'Conv2D': {'strides': [2, 2]}}, {'Pooling2D': {'strides': [2, 2]}},
{'Conv2D': {'strides': [3, 3]}}, {'Pooling2D': {'strides': [3, 3]}},
{'Conv2D': {'strides': [4, 4]}}, {'Pooling2D': {'strides': [4, 4]}}]
assert expected == config.parse(ExperimentSpecification, Parser(), {})
def test_parse_context_expression(self):
parser = Parser()
assert parser.parse_expression(ExperimentSpecification,
'{{ something }}', {}) == ''
assert parser.parse_expression(ExperimentSpecification,
'{{ something }}',
{'something': 1}) == 1
def test_if_operator_config(self):
config_dict = {
'cond': '{{ i }} == 5',
'do': 'It was True',
'else_do': 'It was False'
}
config = IfConfig.from_dict(config_dict)
assert config.to_dict() == config_dict
assert 'It was True' == config.parse(Parser(), {'i': 5})
assert 'It was False' == config.parse(Parser(), {'i': 3})
def __init__(self, values):
self._values = to_list(values)
self._data = reader.read(self._values)
Parser.check_data(spec=self, data=self._data)
headers = Parser.get_headers(spec=self, data=self._data)
try:
self._headers = validator.validate_headers(spec=self, data=headers)
except ValidationError as e:
raise PolyaxonConfigurationError(e)
parsed_data = Parser.parse(self, self._data, None)
self._validated_data = validator.validate(spec=self, data=parsed_data)
self._parsed_data = parsed_data
def test_parse_base_expressions(self):
data = [
1, 10., [1, 1], (1, 1), 'string', ['str1', 'str2'], {
1: 2,
'a': 'a',
'dict': {
1: 1
}
}
]
parser = Parser()
for d in data:
assert d == parser.parse_expression(Specification, d, {})
def test_parse_graph_with_many_inputs_and_non_used_raises(self):
expression = {
'graph': {
'input_layers': ['images', 'sounds'],
'layers': [
{
'Conv2D': {
'filters': 64,
'kernel_size': [3, 3],
'strides': [1, 1],
'activation': 'relu',
'name': 'Conv2D_1',
'inbound_nodes': ['images']
}
},
{
'MaxPooling2D': {
'kernels': 2
}
}, # This layer is orphan
{
'Flatten': {
'inbound_nodes': ['Conv2D_1']
}
}, # Specify the input layer
{
'Dense': {
'units': 10,
'activation': 'softmax',
}
}
]
}
}
parser = Parser()
with self.assertRaises(PolyaxonfileError):
parser.parse_expression(Specification,
expression, {},
check_graph=True)
def get_experiment_spec(self, matrix_declaration):
"""Returns and experiment spec for this group spec and the given matrix declaration."""
parsed_data = Parser.parse(self, self._data, matrix_declaration)
settings = SettingsConfig.get_experiment_settings(
parsed_data[self.SETTINGS])
del parsed_data[self.SETTINGS]
if settings:
parsed_data[self.SETTINGS] = settings
validator.validate(spec=self, data=parsed_data)
return ExperimentSpecification(
values=[parsed_data, {
'kind': self._EXPERIMENT
}])
def __init__(self, values):
self._values = to_list(values)
self._data = reader.read(self._values)
self.check_data()
headers = Parser.get_headers(spec=self, data=self._data)
try:
self._headers = validator.validate_headers(spec=self, data=headers)
except ValidationError as e:
raise PolyaxonConfigurationError(e)
self._parsed_data = None
self._validated_data = None
self._set_parsed_data()
self._extra_validation()
def __init__(self, filepath):
self._filepath = filepath
self._data = reader.read(self._filepath)
self._parsed_data = Parser.parse(self._data)
self._validated_data = validator.validate(self._parsed_data)
def test_parse_graph_expression(self):
expression = {
'graph': {
'input_layers': ['images'],
'layers': [{
'Conv2D': {
'filters': 64,
'kernel_size': [3, 3],
'strides': [1, 1],
'activation': 'relu',
}
}, {
'MaxPooling2D': {
'kernels': 2
}
}, {
'Flatten': None
}, {
'Dense': {
'units': 10,
'activation': 'softmax',
}
}]
}
}
parser = Parser()
assert parser.parse_expression(Specification, expression,
{}) == expression
expected_expression = {
'graph': {
'input_layers': ['images'],
'layers': [{
'Conv2D': {
'activation': 'relu',
'filters': 64,
'kernel_size': [3, 3],
'name': 'Conv2D_1',
'strides': [1, 1],
'inbound_nodes': ['images']
}
}, {
'MaxPooling2D': {
'inbound_nodes': ['Conv2D_1'],
'kernels': 2,
'name': 'MaxPooling2D_1'
}
}, {
'Flatten': {
'inbound_nodes': ['MaxPooling2D_1'],
'name': 'Flatten_1'
}
}, {
'Dense': {
'activation': 'softmax',
'inbound_nodes': ['Flatten_1'],
'name': 'Dense_1',
'units': 10
}
}],
'output_layers': ['Dense_1']
}
}
assert parser.parse_expression(Specification,
expression, {},
check_graph=True) == expected_expression
def test_parse_graph_with_operators_and_tags(self):
declarations = {
'conv2d': {
'filters': [32, 64],
'kernel_size': [[3, 3], [2, 2]],
'strides': [1, 1],
'activation': ['relu', 'linear']
}
}
expression = {
'graph': {
'input_layers': ['images'],
'layers': [{
'for': {
'len': 2,
'index': 'i',
'do': {
'Conv2D': {
'filters': '{{ conv2d.filters[i] }}',
'kernel_size': '{{ conv2d.kernel_size[i] }}',
'strides': '{{ conv2d.strides }}',
'activation': '{{ conv2d.activation[i] }}',
'tags': ['tag1', 'tag2']
}
}
}
}, {
'MaxPooling2D': {
'kernels': 2
}
}, {
'Flatten': {
'name': 'Flatten_1'
}
}, {
'Flatten': {
'inbound_nodes': ['{{ tags.tag1[0] }}'],
'name': 'Flatten_2'
}
}, {
'Concat': {
'inbound_nodes': ['Flatten_1', 'Flatten_2']
}
}, {
'Dense': {
'units': 10,
'activation': 'softmax',
}
}]
}
}
parser = Parser()
result_expression = parser.parse_expression(Specification,
expression,
declarations,
check_operators=True,
check_graph=True)
expected_result = {
'graph': {
'input_layers': ['images'],
'layers': [{
'Conv2D': {
'activation': 'relu',
'filters': 32,
'kernel_size': [3, 3],
'name': 'Conv2D_1',
'strides': [1, 1],
'inbound_nodes': ['images'],
'tags': ['tag1', 'tag2']
}
}, {
'Conv2D': {
'activation': 'linear',
'filters': 64,
'inbound_nodes': ['Conv2D_1'],
'kernel_size': [2, 2],
'name': 'Conv2D_2',
'strides': [1, 1],
'tags': ['tag1', 'tag2']
}
}, {
'MaxPooling2D': {
'inbound_nodes': ['Conv2D_2'],
'kernels': 2,
'name': 'MaxPooling2D_1'
}
}, {
'Flatten': {
'inbound_nodes': ['MaxPooling2D_1'],
'name': 'Flatten_1'
}
}, {
'Flatten': {
'inbound_nodes': ['Conv2D_1'],
'name': 'Flatten_2'
}
}, {
'Concat': {
'inbound_nodes': ['Flatten_1', 'Flatten_2'],
'name': 'Concat_1'
}
}, {
'Dense': {
'activation': 'softmax',
'inbound_nodes': ['Concat_1'],
'name': 'Dense_1',
'units': 10
}
}],
'output_layers': ['Dense_1']
}
}
assert result_expression == expected_result
def _set_parsed_data(self):
parsed_data = Parser.parse(self, self._data, None)
self._validated_data = validator.validate(spec=self, data=parsed_data)
self._parsed_data = parsed_data
def test_parse_context_expression(self):
parser = Parser()
assert parser.parse_expression('{{ something }}', {}) == ''
assert parser.parse_expression('{{ something }}',
{'something': 1}) == 1
def _set_parsed_data(self):
# We need to validate that the data is correct
# For that we just use a matrix declaration test
parsed_data = Parser.parse(self, self._data,
self.matrix_declaration_test)
validator.validate(spec=self, data=parsed_data)