def from_dict(doc: Dict, validate: Optional[bool] = True) -> Parameter:
"""Create a parameter index from a dictionary serialization. Expects a
list of dictionaries, each being a serialized parameter declaration.
Raises an error if parameter indices are not unique.
Parameters
----------
doc: list
List of serialized parameter declarations.
validate: bool, default=True
Validate dictionary serializations if True.
Returns
-------
flowserv.model.template.base.ParameterIndex
"""
parameters = ParameterIndex()
for index, obj in enumerate(doc):
# Ensure that the 'index' property for the parameter is set. Use
# the order of parameters in the list as the default order.
obj['index'] = obj.get('index', index)
# Ensure that the 'isRequired' property is set. If no default value
# is defined the parameter is assumed to be required.
obj['isRequired'] = obj.get('isRequired', 'defaultValue'
not in obj)
para = ParameterDeserializer.from_dict(obj, validate=validate)
if para.name in parameters:
msg = "duplicate parameter '{}'".format(para.name)
raise err.InvalidTemplateError(msg)
parameters[para.name] = para
return parameters
def from_dict(cls, doc, validate=True):
"""Get an instance of the sort column from the dictionary serialization.
Raises an error if the given dictionary does not contain the expected
elements as generated by the to_dict() method of the class.
Parameters
----------
doc: dict
Dictionary serialization of a column object
validate: bool, default=True
Validate the serialization if True.
Returns
-------
flowserv.model.template.schema.SortColumn
Raises
------
flowserv.error.InvalidTemplateError
"""
# Validate the serialization dictionary
if validate:
try:
util.validate_doc(
doc,
mandatory=['name'],
optional=['sortDesc']
)
except ValueError as ex:
raise err.InvalidTemplateError(str(ex))
sort_desc = None
if 'sortDesc' in doc:
sort_desc = doc['sortDesc']
# Return instance of the column object
return cls(column_id=doc['name'], sort_desc=sort_desc)
def get_parameter_references(spec, parameters=None):
"""Get set of parameter identifier that are referenced in the given
workflow specification. Adds parameter identifier to the given parameter
set.
Parameters
----------
spec: dict
Parameterized workflow specification.
parameters: set, optional
Result set of referenced parameter identifier.
Returns
-------
set
Raises
------
flowserv.error.InvalidTemplateError
"""
# Set of referenced parameter identifier.
if parameters is None:
parameters = set()
for key in spec:
val = spec[key]
if isinstance(val, str):
# If the value is of type string extract all parameter references.
for match in re.finditer(REGEX_PARA, val):
# Extract variable name.
parameters.add(get_name(match.group()))
elif isinstance(val, dict):
# Recursive call to get_parameter_references
get_parameter_references(val, parameters=parameters)
elif isinstance(val, list):
for list_val in val:
if isinstance(list_val, str):
# Get potential references to template parameters in
# list elements of type string.
for match in re.finditer(REGEX_PARA, list_val):
# Extract variable name.
parameters.add(get_name(match.group()))
elif isinstance(list_val, dict):
# Recursive replace for dictionaries
get_parameter_references(list_val, parameters=parameters)
elif isinstance(list_val, list):
# We currently do not support lists of lists
raise err.InvalidTemplateError('nested lists not allowed')
return parameters
def __init__(self, parameters: Optional[List[Parameter]] = None):
"""Initialize the parameter index from an optional list of parameters.
Parameters are identified by their name. If the list contains parameters
with identical names an error is raised.
Parameters
----------
parameters: list of flowserv.model.parameter.base.Parameter, default=None
List of parameter declarations that are added to the index.
"""
for para in parameters if parameters is not None else list():
if para.name in self:
msg = "duplicate parameter '{}'".format(para.name)
raise err.InvalidTemplateError(msg)
self[para.name] = para
def replace_args(spec, arguments, parameters):
"""Replace template parameter references in the workflow specification
with their respective values in the argument dictionary or their
defined default value. The type of the result is depending on the type
of the spec object.
Parameters
----------
spec: any
Parameterized workflow specification.
arguments: dict
Dictionary that associates template parameter identifiers with
argument values.
parameters: flowserv.model.template.parameter.ParameterIndex
Dictionary of parameter declarations.
Returns
-------
type(spec)
Raises
------
flowserv.error.InvalidTemplateError
flowserv.error.MissingArgumentError
"""
if isinstance(spec, dict):
# The new object will contain the modified workflow specification
obj = dict()
for key in spec:
obj[key] = replace_args(spec[key], arguments, parameters)
elif isinstance(spec, list):
obj = list()
for val in spec:
if isinstance(val, list):
# We currently do not support lists of lists
raise err.InvalidTemplateError('nested lists not supported')
obj.append(replace_args(val, arguments, parameters))
elif isinstance(spec, str):
obj = expand_value(spec, arguments, parameters)
else:
obj = spec
return obj
def from_dict(cls, doc, validate=True):
"""Get an instance of the column from the dictionary serialization.
Raises an error if the given dictionary does not contain the expected
elements as generated by the to_dict() method of the class.
Parameters
----------
doc: dict
Dictionary serialization of a column object
validate: bool, default=True
Validate the serialization if True.
Returns
-------
flowserv.model.template.schema.ResultColumn
Raises
------
flowserv.error.InvalidTemplateError
"""
# Validate the serialization dictionary.
if validate:
try:
util.validate_doc(
doc,
mandatory=['name', 'label', 'dtype'],
optional=['path', 'required']
)
except ValueError as ex:
raise err.InvalidTemplateError(str(ex))
# Return instance of the column object
return cls(
column_id=doc['name'],
name=doc['label'],
dtype=doc['dtype'],
path=doc.get('path'),
required=doc.get('required')
)
def __init__(self, column_id, name, dtype, path=None, required=None):
"""Initialize the unique column identifier, name, and the data type. If
the value of dtype is not in the list of supported data types an
error is raised.
The optional path element references the column value in nested result
files. If no path is given the column identifier is used instead.
Parameters
----------
column_id: string
Unique column identifier
name: string
Unique column name
dtype: string
Data type identifier
path: string, optional
Path to column value in nested result files.
required: bool, optional
Indicates whether a value is expected for this column in every
benchmark run result
Raises
------
flowserv.error.InvalidTemplateError
"""
# Raise error if the data type value is not in the list of supported
# data types
if dtype not in DATA_TYPES:
msg = "unknown data type '{}'"
raise err.InvalidTemplateError(msg.format(dtype))
self.column_id = column_id
self.name = name
self.dtype = dtype
self.path = path
self.required = required if required is not None else True
def from_dict(cls, doc, validate=True):
"""Get an instance of the schema from a dictionary serialization.
Raises an error if the given dictionary does not contain the expected
elements as generated by the to_dict() method of the class or if the
names or identifier of columns are not unique.
Returns None if the given document is None.
Parameters
----------
doc: dict
Dictionary serialization of a benchmark result schema object
validate: bool, default=True
Validate the serialization if True.
Returns
-------
flowserv.model.template.schema.ResultSchema
Raises
------
flowserv.error.InvalidTemplateError
"""
# Return None if no document is given
if doc is None:
return None
# Validate the serialization dictionary
if validate:
try:
util.validate_doc(
doc,
mandatory=['file', 'schema'],
optional=['orderBy']
)
except ValueError as ex:
raise err.InvalidTemplateError(str(ex))
# Identifier of the output file that contains the result object
file_id = doc['file']
# Get column list. Ensure that all column names and identifier are
# unique
columns = list()
for c in doc['schema']:
columns.append(ResultColumn.from_dict(c, validate=validate))
ids = set()
names = set()
for col in columns:
if col.column_id in ids:
msg = "duplicate column identifier '{}'"
raise err.InvalidTemplateError(msg.format(col.column_id))
ids.add(col.column_id)
if col.name in names:
msg = "not unique column name '{}'"
raise err.InvalidTemplateError(msg.format(col.name))
names.add(col.name)
# Get optional default sort statement for the ranking
order_by = list()
for c in doc.get('orderBy', []):
col = SortColumn.from_dict(c, validate=validate)
if col.column_id not in ids:
msg = "unknown column '{}'"
raise err.InvalidTemplateError(msg.format(col.column_id))
order_by.append(col)
# Return benchmark schema object
return cls(
result_file=file_id,
columns=columns,
order_by=order_by
)
def from_dict(cls, doc, validate=True):
"""Create an instance of the workflow template for a dictionary
serialization. The structure of the dictionary is expected to be the
same as generated by the to_dict() method of this class. The only
mandatory element in the dictionary is the workflow specification.
Parameters
----------
doc: dict
Dictionary serialization of a workflow template
validate: bool, optional
Validate template parameter declarations against the parameter
schema if this flag is True.
Returns
-------
flowserv.model.template.base.WorkflowTemplate
Raises
------
flowserv.error.InvalidTemplateError
flowserv.error.UnknownParameterError
"""
# Ensure that the mandatory elements are present. At this point, only
# the workflow specification is mandatory.
if validate:
if 'workflow' not in doc:
msg = "missing element '{}'".format('workflow')
raise err.InvalidTemplateError(msg)
# -- Workflow specification -------------------------------------------
workflow_spec = doc['workflow']
# -- Parameter declarations -------------------------------------------
# Add given parameter declarations to the parameter list. Ensure that
# all default values are set
parameters = ParameterIndex.from_dict(
doc.get('parameters', dict()),
validate=validate
)
# Ensure that the workflow specification does not reference
# undefined parameters if validate flag is True.
if validate:
for key in tp.get_parameter_references(workflow_spec):
if key not in parameters:
raise err.UnknownParameterError(key)
# -- Post-processing task ---------------------------------------------
postproc_spec = None
if 'postproc' in doc:
postproc_spec = doc['postproc']
if validate:
util.validate_doc(
doc=postproc_spec,
mandatory=['workflow'],
optional=['inputs', 'outputs']
)
util.validate_doc(
doc=postproc_spec.get('inputs', {'files': ''}),
mandatory=['files'],
optional=['runs']
)
# -- Parameter module information -------------------------------------
parameter_groups = None
if 'parameterGroups' in doc:
parameter_groups = list()
for m in doc['parameterGroups']:
parameter_groups.append(ParameterGroup.from_dict(m, validate=validate))
# -- Output file specifications --------------------------------------
outputs = None
if 'outputs' in doc:
outputs = [WorkflowOutputFile.from_dict(
f,
validate=validate
) for f in doc['outputs']]
# -- Result schema ---------------------------------------------------
schema = ResultSchema.from_dict(doc.get('results'), validate=validate)
# Return template instance
return cls(
workflow_spec=workflow_spec,
postproc_spec=postproc_spec,
parameters=parameters,
result_schema=schema,
parameter_groups=parameter_groups,
outputs=outputs
)