def __init__(self, values): self._values = to_list(values) self._data = reader.read(self._values) Parser.check_data(spec=self, data=self._data) headers = Parser.get_headers(spec=self, data=self._data) matrix = Parser.get_matrix(spec=self, data=self._data) try: self._matrix = validator.validate_matrix(matrix) except ValidationError as e: raise PolyaxonConfigurationError(e) try: self._headers = validator.validate_headers(spec=self, data=headers) except ValidationError as e: raise PolyaxonConfigurationError(e) self._parsed_data = [] self._validated_data = [] self._experiment_specs = [] matrix_declarations = self.matrix_declarations if self.matrix_declarations else [ {} ] for i, matrix_declaration in enumerate(matrix_declarations): parsed_data = Parser.parse(self, self._data, matrix_declaration) self._validated_data.append( validator.validate(spec=self, data=parsed_data)) self._parsed_data.append(parsed_data) self._experiment_specs.append( Specification(experiment=i, values=parsed_data))
def test_for_operator_config(self): config_dict = { 'len': 5, 'do': 'Value at {{ i }}', 'index': 'i' } config = ForConfig.from_dict(config_dict) assert config.to_dict() == config_dict expected = ['Value at 0', 'Value at 1', 'Value at 2', 'Value at 3', 'Value at 4'] assert expected == config.parse(ExperimentSpecification, Parser(), {}) config_dict = { 'len': 5, 'do': [{'Conv2D': {'strides': ['{{ i }}', '{{ i }}']}}, {'Pooling2D': {'strides': ['{{ i }}', '{{ i }}']}}], 'index': 'i' } config = ForConfig.from_dict(config_dict) assert config.to_dict() == config_dict # Lists get flattened expected = [{'Conv2D': {'strides': [0, 0]}}, {'Pooling2D': {'strides': [0, 0]}}, {'Conv2D': {'strides': [1, 1]}}, {'Pooling2D': {'strides': [1, 1]}}, {'Conv2D': {'strides': [2, 2]}}, {'Pooling2D': {'strides': [2, 2]}}, {'Conv2D': {'strides': [3, 3]}}, {'Pooling2D': {'strides': [3, 3]}}, {'Conv2D': {'strides': [4, 4]}}, {'Pooling2D': {'strides': [4, 4]}}] assert expected == config.parse(ExperimentSpecification, Parser(), {})
def test_parse_context_expression(self): parser = Parser() assert parser.parse_expression(ExperimentSpecification, '{{ something }}', {}) == '' assert parser.parse_expression(ExperimentSpecification, '{{ something }}', {'something': 1}) == 1
def test_if_operator_config(self): config_dict = { 'cond': '{{ i }} == 5', 'do': 'It was True', 'else_do': 'It was False' } config = IfConfig.from_dict(config_dict) assert config.to_dict() == config_dict assert 'It was True' == config.parse(Parser(), {'i': 5}) assert 'It was False' == config.parse(Parser(), {'i': 3})
def __init__(self, values): self._values = to_list(values) self._data = reader.read(self._values) Parser.check_data(spec=self, data=self._data) headers = Parser.get_headers(spec=self, data=self._data) try: self._headers = validator.validate_headers(spec=self, data=headers) except ValidationError as e: raise PolyaxonConfigurationError(e) parsed_data = Parser.parse(self, self._data, None) self._validated_data = validator.validate(spec=self, data=parsed_data) self._parsed_data = parsed_data
def test_parse_base_expressions(self): data = [ 1, 10., [1, 1], (1, 1), 'string', ['str1', 'str2'], { 1: 2, 'a': 'a', 'dict': { 1: 1 } } ] parser = Parser() for d in data: assert d == parser.parse_expression(Specification, d, {})
def test_parse_graph_with_many_inputs_and_non_used_raises(self): expression = { 'graph': { 'input_layers': ['images', 'sounds'], 'layers': [ { 'Conv2D': { 'filters': 64, 'kernel_size': [3, 3], 'strides': [1, 1], 'activation': 'relu', 'name': 'Conv2D_1', 'inbound_nodes': ['images'] } }, { 'MaxPooling2D': { 'kernels': 2 } }, # This layer is orphan { 'Flatten': { 'inbound_nodes': ['Conv2D_1'] } }, # Specify the input layer { 'Dense': { 'units': 10, 'activation': 'softmax', } } ] } } parser = Parser() with self.assertRaises(PolyaxonfileError): parser.parse_expression(Specification, expression, {}, check_graph=True)
def get_experiment_spec(self, matrix_declaration): """Returns and experiment spec for this group spec and the given matrix declaration.""" parsed_data = Parser.parse(self, self._data, matrix_declaration) settings = SettingsConfig.get_experiment_settings( parsed_data[self.SETTINGS]) del parsed_data[self.SETTINGS] if settings: parsed_data[self.SETTINGS] = settings validator.validate(spec=self, data=parsed_data) return ExperimentSpecification( values=[parsed_data, { 'kind': self._EXPERIMENT }])
def __init__(self, values): self._values = to_list(values) self._data = reader.read(self._values) self.check_data() headers = Parser.get_headers(spec=self, data=self._data) try: self._headers = validator.validate_headers(spec=self, data=headers) except ValidationError as e: raise PolyaxonConfigurationError(e) self._parsed_data = None self._validated_data = None self._set_parsed_data() self._extra_validation()
def __init__(self, filepath): self._filepath = filepath self._data = reader.read(self._filepath) self._parsed_data = Parser.parse(self._data) self._validated_data = validator.validate(self._parsed_data)
def test_parse_graph_expression(self): expression = { 'graph': { 'input_layers': ['images'], 'layers': [{ 'Conv2D': { 'filters': 64, 'kernel_size': [3, 3], 'strides': [1, 1], 'activation': 'relu', } }, { 'MaxPooling2D': { 'kernels': 2 } }, { 'Flatten': None }, { 'Dense': { 'units': 10, 'activation': 'softmax', } }] } } parser = Parser() assert parser.parse_expression(Specification, expression, {}) == expression expected_expression = { 'graph': { 'input_layers': ['images'], 'layers': [{ 'Conv2D': { 'activation': 'relu', 'filters': 64, 'kernel_size': [3, 3], 'name': 'Conv2D_1', 'strides': [1, 1], 'inbound_nodes': ['images'] } }, { 'MaxPooling2D': { 'inbound_nodes': ['Conv2D_1'], 'kernels': 2, 'name': 'MaxPooling2D_1' } }, { 'Flatten': { 'inbound_nodes': ['MaxPooling2D_1'], 'name': 'Flatten_1' } }, { 'Dense': { 'activation': 'softmax', 'inbound_nodes': ['Flatten_1'], 'name': 'Dense_1', 'units': 10 } }], 'output_layers': ['Dense_1'] } } assert parser.parse_expression(Specification, expression, {}, check_graph=True) == expected_expression
def test_parse_graph_with_operators_and_tags(self): declarations = { 'conv2d': { 'filters': [32, 64], 'kernel_size': [[3, 3], [2, 2]], 'strides': [1, 1], 'activation': ['relu', 'linear'] } } expression = { 'graph': { 'input_layers': ['images'], 'layers': [{ 'for': { 'len': 2, 'index': 'i', 'do': { 'Conv2D': { 'filters': '{{ conv2d.filters[i] }}', 'kernel_size': '{{ conv2d.kernel_size[i] }}', 'strides': '{{ conv2d.strides }}', 'activation': '{{ conv2d.activation[i] }}', 'tags': ['tag1', 'tag2'] } } } }, { 'MaxPooling2D': { 'kernels': 2 } }, { 'Flatten': { 'name': 'Flatten_1' } }, { 'Flatten': { 'inbound_nodes': ['{{ tags.tag1[0] }}'], 'name': 'Flatten_2' } }, { 'Concat': { 'inbound_nodes': ['Flatten_1', 'Flatten_2'] } }, { 'Dense': { 'units': 10, 'activation': 'softmax', } }] } } parser = Parser() result_expression = parser.parse_expression(Specification, expression, declarations, check_operators=True, check_graph=True) expected_result = { 'graph': { 'input_layers': ['images'], 'layers': [{ 'Conv2D': { 'activation': 'relu', 'filters': 32, 'kernel_size': [3, 3], 'name': 'Conv2D_1', 'strides': [1, 1], 'inbound_nodes': ['images'], 'tags': ['tag1', 'tag2'] } }, { 'Conv2D': { 'activation': 'linear', 'filters': 64, 'inbound_nodes': ['Conv2D_1'], 'kernel_size': [2, 2], 'name': 'Conv2D_2', 'strides': [1, 1], 'tags': ['tag1', 'tag2'] } }, { 'MaxPooling2D': { 'inbound_nodes': ['Conv2D_2'], 'kernels': 2, 'name': 'MaxPooling2D_1' } }, { 'Flatten': { 'inbound_nodes': ['MaxPooling2D_1'], 'name': 'Flatten_1' } }, { 'Flatten': { 'inbound_nodes': ['Conv2D_1'], 'name': 'Flatten_2' } }, { 'Concat': { 'inbound_nodes': ['Flatten_1', 'Flatten_2'], 'name': 'Concat_1' } }, { 'Dense': { 'activation': 'softmax', 'inbound_nodes': ['Concat_1'], 'name': 'Dense_1', 'units': 10 } }], 'output_layers': ['Dense_1'] } } assert result_expression == expected_result
def _set_parsed_data(self): parsed_data = Parser.parse(self, self._data, None) self._validated_data = validator.validate(spec=self, data=parsed_data) self._parsed_data = parsed_data
def test_parse_context_expression(self): parser = Parser() assert parser.parse_expression('{{ something }}', {}) == '' assert parser.parse_expression('{{ something }}', {'something': 1}) == 1
def _set_parsed_data(self): # We need to validate that the data is correct # For that we just use a matrix declaration test parsed_data = Parser.parse(self, self._data, self.matrix_declaration_test) validator.validate(spec=self, data=parsed_data)