def validate(data):
"""Validates the data and creates the config objects"""
if 'project' not in data:
raise PolyaxonfileError(
"The Polyaxonfile must contain a project section.")
if 'model' not in data:
raise PolyaxonfileError(
"The Polyaxonfile must contain a model section.")
validated_data = {
'version': data['version'],
'project': ProjectConfig.from_dict(data['project']),
'model': ModelConfig.from_dict(data['model'])
}
if data.get('settings'):
validated_data['settings'] = SettingsConfig.from_dict(data['settings'])
if data.get('train'):
validated_data['train'] = TrainConfig.from_dict(data['train'])
if data.get('eval'):
validated_data['eval'] = EvalConfig.from_dict(data['eval'])
return validated_data
def __init__(self, filepaths, params=None, debug_ttl=False):
filepaths = to_list(filepaths)
for filepath in filepaths:
if not os.path.isfile(filepath):
raise PolyaxonfileError(
"`{}` must be a valid file".format(filepath))
self._filenames = [
os.path.basename(filepath) for filepath in filepaths
]
if params:
if not isinstance(params, Mapping):
raise PolyaxonfileError(
"Params: `{}` must be a valid mapping".format(params))
filepaths.append({'params': params})
if debug_ttl:
if not isinstance(debug_ttl, int):
raise PolyaxonfileError(
"Debug TTL `{}` must be a valid integer".format(debug_ttl))
filepaths.append({'run': {'cmd': 'sleep {}'.format(debug_ttl)}})
data = rhea.read(filepaths)
kind = BaseSpecification.get_kind(data=data)
debug_cond = (debug_ttl
and not (BaseSpecification.check_kind_experiment(kind)
or BaseSpecification.check_kind_job(kind)))
if debug_cond:
raise PolyaxonfileError(
'You can only trigger debug mode on a job or an experiment specification, '
'received instead a `{}` specification'.format(kind))
try:
self.specification = SPECIFICATION_BY_KIND[kind](data)
except PolyaxonConfigurationError as e:
raise PolyaxonfileError(e)
def check_data(self, data=None):
data = data or self._data
self.check_version(data)
self.check_kind(data)
if data[self.KIND] != self._SPEC_KIND:
raise PolyaxonfileError(
"The specification used `{}` is incompatible with the kind `{}`."
.format(self.__class__.__name__, data[self.KIND]))
for key in set(six.iterkeys(data)) - set(self.SECTIONS):
raise PolyaxonfileError(
"Unexpected section `{}` in Polyaxonfile version `{}`. "
"Please check the Polyaxonfile specification "
"for this version.".format(key, data[self.VERSION]))
for key in set(six.iterkeys(data)) - set(self.POSSIBLE_SECTIONS):
raise PolyaxonfileError(
"Unexpected section `{}` for specification kind `{}` version `{}`. "
"Please check the Polyaxonfile specification "
"for this version.".format(key, self._SPEC_KIND,
data[self.VERSION]))
for key in self.REQUIRED_SECTIONS:
if key not in data:
raise PolyaxonfileError(
"{} is a required section for a valid Polyaxonfile".format(
key))
def check_kind(cls, data):
if cls.KIND not in data:
raise PolyaxonfileError(
"The Polyaxonfile `kind` must be specified.")
if data[cls.KIND] not in cls._KINDS:
raise PolyaxonfileError(
"The Polyaxonfile with kind `{}` is not a supported value.".
format(data[cls.KIND]))
def validate_version(spec, data):
if 'version' not in data:
raise PolyaxonfileError("The Polyaxonfile version must be specified.")
if not (spec.MIN_VERSION <= data['version'] <= spec.MAX_VERSION):
raise PolyaxonfileError(
"The Polyaxonfile's version specified is not supported by your current CLI."
"Your CLI support Polyaxonfile versions between: {} {}."
"You can run `polyaxon upgrade` and "
"check documentation for the specification.".format(
spec.MIN_VERSION, spec.MAX_VERSION))
def check_version(cls, data):
if cls.VERSION not in data:
raise PolyaxonfileError(
"The Polyaxonfile `version` must be specified.")
if not cls.MIN_VERSION <= data[cls.VERSION] <= cls.MAX_VERSION:
raise PolyaxonfileError(
"The Polyaxonfile's version specified is not supported by your current CLI."
"Your CLI support Polyaxonfile versions between: {} {}."
"You can run `polyaxon upgrade` and "
"check documentation for the specification.".format(
cls.MIN_VERSION, cls.MAX_VERSION))
def check_data(cls, spec, data):
cls.validate_version(spec, data)
for key in (set(six.iterkeys(data)) - set(spec.SECTIONS)):
raise PolyaxonfileError("Unexpected section `{}` in Polyaxonfile version `{}`. "
"Please check the Polyaxonfile specification "
"for this version.".format(key, 'v1'))
for key in spec.REQUIRED_SECTIONS:
if key not in data:
raise PolyaxonfileError("{} is a required section for a valid Polyaxonfile".format(
key
))
def __init__(self, filepaths):
filepaths = to_list(filepaths)
for filepath in filepaths:
if not os.path.isfile(filepath):
raise PolyaxonfileError(
"`{}` must be a valid file".format(filepath))
self._filenames = [
os.path.basename(filepath) for filepath in filepaths
]
try:
super(PolyaxonFile, self).__init__(filepaths)
except PolyaxonConfigurationError as e:
raise PolyaxonfileError(e)
def __init__(self, filepaths):
filepaths = to_list(filepaths)
for filepath in filepaths:
if not os.path.isfile(filepath):
raise PolyaxonfileError(
"`{}` must be a valid file".format(filepath))
self._filenames = [
os.path.basename(filepath) for filepath in filepaths
]
data = reader.read(filepaths)
kind = BaseSpecification.get_kind(data=data)
try:
self.specification = SPECIFICATION_BY_KIND[kind](data)
except PolyaxonConfigurationError as e:
raise PolyaxonfileError(e)
def parse(cls, data):
cls.validate_version(data)
sections = Specification.sections()
for key in (set(six.iterkeys(data)) - sections):
raise PolyaxonfileError(
"Unexpected section `{}` in Polyaxonfile version `{}`."
"Please check the Polyaxonfile specification "
"for this version.".format(key, 'v1'))
parsed_data = {
'version': data['version'],
}
if 'declarations' in data:
parsed_data['declarations'] = cls.parse_expression(
data['declarations'], data['declarations'])
if 'matrix' in data:
parsed_data['matrix'] = cls.parse_expression(
data['matrix'], parsed_data.get('declarations'))
for section in Specification.SECTIONS:
if section in data:
parsed_data[section] = cls.parse_expression(
data[section], parsed_data.get('declarations', {}))
for section in Specification.GRAPH_SECTIONS:
if section in data:
parsed_data[section] = cls.parse_expression(
data[section], parsed_data.get('declarations', {}), True,
True)
return parsed_data
def validate_keys(section, config, section_data):
extra_args = [
key for key in section_data.keys()
if key not in config.SCHEMA().fields
]
if extra_args:
raise PolyaxonfileError(
'Extra arguments passed for `{}`: {}'.format(
section, extra_args))
def validate_keys(section, config, section_data):
if not isinstance(section_data, dict) or section == spec.MODEL:
return
extra_args = [
key for key in section_data.keys()
if key not in config.SCHEMA().fields
]
if extra_args:
raise PolyaxonfileError(
'Extra arguments passed for `{}`: {}'.format(
section, extra_args))
def _parse_graph(cls, spec, graph,
declarations): # noqa, too-many-branches
input_layers = to_list(graph['input_layers'])
layer_names = set(input_layers)
tags = {}
layers = []
outputs = []
layers_counters = defaultdict(int)
unused_layers = set(input_layers)
if not isinstance(graph['layers'], list):
raise PolyaxonfileError(
"Graph definition expects a list of layer definitions.")
def add_tag(tag, layer_value):
if tag in tags:
tags[tag] = to_list(tags[tag])
tags[tag].append(layer_value['name'])
else:
tags[tag] = layer_value['name']
def get_layer_name(layer_value, layer_type):
if 'name' not in layer_value:
layers_counters[layer_type] += 1
return '{}_{}'.format(layer_type, layers_counters[layer_type])
return layer_value['name']
layers_declarations = {}
layers_declarations.update(declarations)
last_layer = None
first_layer = True
for layer_expression in graph['layers']:
parsed_layer = cls.parse_expression(spec, layer_expression,
layers_declarations, True)
# Gather all tags from the layers
parsed_layer = to_list(parsed_layer)
for layer in parsed_layer:
if not layer:
continue
layer_type, layer_value = list(six.iteritems(layer))[0]
if layer_value is None:
layer_value = {}
# Check that the layer has a name otherwise generate one
name = get_layer_name(layer_value, layer_type)
if name not in layer_names:
layer_names.add(name)
layer_value['name'] = name
else:
raise PolyaxonfileError(
"The name `{}` is used 2 times in the graph. "
"All layer names should be unique. "
"If you need to reference a layer in a for loop "
"think about using `tags`".format(name))
for tag in to_list(layer_value.get('tags', [])):
add_tag(tag, layer_value)
# Check if the layer is an output
if layer_value.get('is_output', False) is True:
outputs.append(layer_value['name'])
else:
# Add the layer to unused
unused_layers.add(layer_value['name'])
# Check the layers inputs
if not layer_value.get('inbound_nodes'):
if last_layer is not None:
layer_value['inbound_nodes'] = [
last_layer['name']
] # noqa, unsubscriptable-object
if first_layer and len(input_layers) == 1:
layer_value['inbound_nodes'] = input_layers
if first_layer and len(input_layers) > 1:
raise PolyaxonfileError(
"The first layer must indicate which input to use,"
"You have {} layers: {}".format(
len(input_layers), input_layers))
first_layer = False
for input_layer in layer_value.get('inbound_nodes', []):
if input_layer not in layer_names:
raise PolyaxonfileError(
"The layer `{}` has a non existing "
"inbound node `{}`".format(layer_value['name'],
input_layer))
if input_layer in unused_layers:
unused_layers.remove(input_layer)
# Add layer
layers.append({layer_type: layer_value})
# Update layers_declarations
layers_declarations['tags'] = tags
# Update last_layer
last_layer = layer_value
# Add last layer as output
if last_layer:
if last_layer['name'] not in outputs:
outputs.append(last_layer['name'])
# Remove last layer from unused layers
if last_layer['name'] in unused_layers:
unused_layers.remove(last_layer['name'])
# Check if some layers are unused
if unused_layers:
raise PolyaxonfileError(
"These layers `{}` were declared but are not used.".format(
unused_layers))
return {
'input_layers': to_list(graph['input_layers']),
'layers': layers,
'output_layers': outputs
}