Пример #1
0
def validate(data):
    """Validates the data and creates the config objects"""
    if 'project' not in data:
        raise PolyaxonfileError(
            "The Polyaxonfile must contain a project section.")

    if 'model' not in data:
        raise PolyaxonfileError(
            "The Polyaxonfile must contain a model section.")

    validated_data = {
        'version': data['version'],
        'project': ProjectConfig.from_dict(data['project']),
        'model': ModelConfig.from_dict(data['model'])
    }
    if data.get('settings'):
        validated_data['settings'] = SettingsConfig.from_dict(data['settings'])

    if data.get('train'):
        validated_data['train'] = TrainConfig.from_dict(data['train'])

    if data.get('eval'):
        validated_data['eval'] = EvalConfig.from_dict(data['eval'])

    return validated_data
Пример #2
0
    def __init__(self, filepaths, params=None, debug_ttl=False):
        filepaths = to_list(filepaths)
        for filepath in filepaths:
            if not os.path.isfile(filepath):
                raise PolyaxonfileError(
                    "`{}` must be a valid file".format(filepath))
        self._filenames = [
            os.path.basename(filepath) for filepath in filepaths
        ]
        if params:
            if not isinstance(params, Mapping):
                raise PolyaxonfileError(
                    "Params: `{}` must be a valid mapping".format(params))
            filepaths.append({'params': params})
        if debug_ttl:
            if not isinstance(debug_ttl, int):
                raise PolyaxonfileError(
                    "Debug TTL `{}` must be a valid integer".format(debug_ttl))
            filepaths.append({'run': {'cmd': 'sleep {}'.format(debug_ttl)}})
        data = rhea.read(filepaths)
        kind = BaseSpecification.get_kind(data=data)

        debug_cond = (debug_ttl
                      and not (BaseSpecification.check_kind_experiment(kind)
                               or BaseSpecification.check_kind_job(kind)))
        if debug_cond:
            raise PolyaxonfileError(
                'You can only trigger debug mode on a job or an experiment specification, '
                'received instead a `{}` specification'.format(kind))
        try:
            self.specification = SPECIFICATION_BY_KIND[kind](data)
        except PolyaxonConfigurationError as e:
            raise PolyaxonfileError(e)
Пример #3
0
    def check_data(self, data=None):
        data = data or self._data
        self.check_version(data)
        self.check_kind(data)
        if data[self.KIND] != self._SPEC_KIND:
            raise PolyaxonfileError(
                "The specification used `{}` is incompatible with the kind `{}`."
                .format(self.__class__.__name__, data[self.KIND]))
        for key in set(six.iterkeys(data)) - set(self.SECTIONS):
            raise PolyaxonfileError(
                "Unexpected section `{}` in Polyaxonfile version `{}`. "
                "Please check the Polyaxonfile specification "
                "for this version.".format(key, data[self.VERSION]))

        for key in set(six.iterkeys(data)) - set(self.POSSIBLE_SECTIONS):
            raise PolyaxonfileError(
                "Unexpected section `{}` for specification kind `{}` version `{}`. "
                "Please check the Polyaxonfile specification "
                "for this version.".format(key, self._SPEC_KIND,
                                           data[self.VERSION]))

        for key in self.REQUIRED_SECTIONS:
            if key not in data:
                raise PolyaxonfileError(
                    "{} is a required section for a valid Polyaxonfile".format(
                        key))
Пример #4
0
    def check_kind(cls, data):
        if cls.KIND not in data:
            raise PolyaxonfileError(
                "The Polyaxonfile `kind` must be specified.")

        if data[cls.KIND] not in cls._KINDS:
            raise PolyaxonfileError(
                "The Polyaxonfile with kind `{}` is not a supported value.".
                format(data[cls.KIND]))
Пример #5
0
 def validate_version(spec, data):
     if 'version' not in data:
         raise PolyaxonfileError("The Polyaxonfile version must be specified.")
     if not (spec.MIN_VERSION <= data['version'] <= spec.MAX_VERSION):
         raise PolyaxonfileError(
             "The Polyaxonfile's version specified is not supported by your current CLI."
             "Your CLI support Polyaxonfile versions between: {} {}."
             "You can run `polyaxon upgrade` and "
             "check documentation for the specification.".format(
                 spec.MIN_VERSION, spec.MAX_VERSION))
Пример #6
0
 def check_version(cls, data):
     if cls.VERSION not in data:
         raise PolyaxonfileError(
             "The Polyaxonfile `version` must be specified.")
     if not cls.MIN_VERSION <= data[cls.VERSION] <= cls.MAX_VERSION:
         raise PolyaxonfileError(
             "The Polyaxonfile's version specified is not supported by your current CLI."
             "Your CLI support Polyaxonfile versions between: {} {}."
             "You can run `polyaxon upgrade` and "
             "check documentation for the specification.".format(
                 cls.MIN_VERSION, cls.MAX_VERSION))
Пример #7
0
    def check_data(cls, spec, data):
        cls.validate_version(spec, data)
        for key in (set(six.iterkeys(data)) - set(spec.SECTIONS)):
            raise PolyaxonfileError("Unexpected section `{}` in Polyaxonfile version `{}`. "
                                    "Please check the Polyaxonfile specification "
                                    "for this version.".format(key, 'v1'))

        for key in spec.REQUIRED_SECTIONS:
            if key not in data:
                raise PolyaxonfileError("{} is a required section for a valid Polyaxonfile".format(
                    key
                ))
 def __init__(self, filepaths):
     filepaths = to_list(filepaths)
     for filepath in filepaths:
         if not os.path.isfile(filepath):
             raise PolyaxonfileError(
                 "`{}` must be a valid file".format(filepath))
     self._filenames = [
         os.path.basename(filepath) for filepath in filepaths
     ]
     try:
         super(PolyaxonFile, self).__init__(filepaths)
     except PolyaxonConfigurationError as e:
         raise PolyaxonfileError(e)
Пример #9
0
 def __init__(self, filepaths):
     filepaths = to_list(filepaths)
     for filepath in filepaths:
         if not os.path.isfile(filepath):
             raise PolyaxonfileError(
                 "`{}` must be a valid file".format(filepath))
     self._filenames = [
         os.path.basename(filepath) for filepath in filepaths
     ]
     data = reader.read(filepaths)
     kind = BaseSpecification.get_kind(data=data)
     try:
         self.specification = SPECIFICATION_BY_KIND[kind](data)
     except PolyaxonConfigurationError as e:
         raise PolyaxonfileError(e)
Пример #10
0
    def parse(cls, data):
        cls.validate_version(data)
        sections = Specification.sections()
        for key in (set(six.iterkeys(data)) - sections):
            raise PolyaxonfileError(
                "Unexpected section `{}` in Polyaxonfile version `{}`."
                "Please check the Polyaxonfile specification "
                "for this version.".format(key, 'v1'))

        parsed_data = {
            'version': data['version'],
        }

        if 'declarations' in data:
            parsed_data['declarations'] = cls.parse_expression(
                data['declarations'], data['declarations'])

        if 'matrix' in data:
            parsed_data['matrix'] = cls.parse_expression(
                data['matrix'], parsed_data.get('declarations'))

        for section in Specification.SECTIONS:
            if section in data:
                parsed_data[section] = cls.parse_expression(
                    data[section], parsed_data.get('declarations', {}))

        for section in Specification.GRAPH_SECTIONS:
            if section in data:
                parsed_data[section] = cls.parse_expression(
                    data[section], parsed_data.get('declarations', {}), True,
                    True)

        return parsed_data
Пример #11
0
 def validate_keys(section, config, section_data):
     extra_args = [
         key for key in section_data.keys()
         if key not in config.SCHEMA().fields
     ]
     if extra_args:
         raise PolyaxonfileError(
             'Extra arguments passed for `{}`: {}'.format(
                 section, extra_args))
Пример #12
0
    def validate_keys(section, config, section_data):
        if not isinstance(section_data, dict) or section == spec.MODEL:
            return

        extra_args = [
            key for key in section_data.keys()
            if key not in config.SCHEMA().fields
        ]
        if extra_args:
            raise PolyaxonfileError(
                'Extra arguments passed for `{}`: {}'.format(
                    section, extra_args))
Пример #13
0
    def _parse_graph(cls, spec, graph,
                     declarations):  # noqa, too-many-branches
        input_layers = to_list(graph['input_layers'])
        layer_names = set(input_layers)
        tags = {}
        layers = []
        outputs = []
        layers_counters = defaultdict(int)
        unused_layers = set(input_layers)

        if not isinstance(graph['layers'], list):
            raise PolyaxonfileError(
                "Graph definition expects a list of layer definitions.")

        def add_tag(tag, layer_value):
            if tag in tags:
                tags[tag] = to_list(tags[tag])
                tags[tag].append(layer_value['name'])
            else:
                tags[tag] = layer_value['name']

        def get_layer_name(layer_value, layer_type):
            if 'name' not in layer_value:
                layers_counters[layer_type] += 1
                return '{}_{}'.format(layer_type, layers_counters[layer_type])

            return layer_value['name']

        layers_declarations = {}
        layers_declarations.update(declarations)

        last_layer = None
        first_layer = True
        for layer_expression in graph['layers']:
            parsed_layer = cls.parse_expression(spec, layer_expression,
                                                layers_declarations, True)
            # Gather all tags from the layers
            parsed_layer = to_list(parsed_layer)
            for layer in parsed_layer:
                if not layer:
                    continue

                layer_type, layer_value = list(six.iteritems(layer))[0]

                if layer_value is None:
                    layer_value = {}
                # Check that the layer has a name otherwise generate one
                name = get_layer_name(layer_value, layer_type)
                if name not in layer_names:
                    layer_names.add(name)
                    layer_value['name'] = name
                else:
                    raise PolyaxonfileError(
                        "The name `{}` is used 2 times in the graph. "
                        "All layer names should be unique. "
                        "If you need to reference a layer in a for loop "
                        "think about using `tags`".format(name))

                for tag in to_list(layer_value.get('tags', [])):
                    add_tag(tag, layer_value)

                # Check if the layer is an output
                if layer_value.get('is_output', False) is True:
                    outputs.append(layer_value['name'])
                else:
                    # Add the layer to unused
                    unused_layers.add(layer_value['name'])

                # Check the layers inputs
                if not layer_value.get('inbound_nodes'):
                    if last_layer is not None:
                        layer_value['inbound_nodes'] = [
                            last_layer['name']
                        ]  # noqa, unsubscriptable-object
                    if first_layer and len(input_layers) == 1:
                        layer_value['inbound_nodes'] = input_layers
                    if first_layer and len(input_layers) > 1:
                        raise PolyaxonfileError(
                            "The first layer must indicate which input to use,"
                            "You have {} layers: {}".format(
                                len(input_layers), input_layers))

                first_layer = False
                for input_layer in layer_value.get('inbound_nodes', []):
                    if input_layer not in layer_names:
                        raise PolyaxonfileError(
                            "The layer `{}` has a non existing "
                            "inbound node `{}`".format(layer_value['name'],
                                                       input_layer))
                    if input_layer in unused_layers:
                        unused_layers.remove(input_layer)

                # Add layer
                layers.append({layer_type: layer_value})

                # Update layers_declarations
                layers_declarations['tags'] = tags

                # Update last_layer
                last_layer = layer_value

        # Add last layer as output
        if last_layer:
            if last_layer['name'] not in outputs:
                outputs.append(last_layer['name'])

            # Remove last layer from unused layers
            if last_layer['name'] in unused_layers:
                unused_layers.remove(last_layer['name'])

        # Check if some layers are unused
        if unused_layers:
            raise PolyaxonfileError(
                "These layers `{}` were declared but are not used.".format(
                    unused_layers))

        return {
            'input_layers': to_list(graph['input_layers']),
            'layers': layers,
            'output_layers': outputs
        }