def annotate(components, lazy=True):
schemas = dict()
components = list_to_dict(components)
for name in components:
component = components[name]
component_type = component["meta"]["type"]
validator = validation.get(component)
try:
if component_type not in schemas:
schemas[component_type] = validator.load_schema(component_type)
if schemas[component_type] is not None:
validator.validate_schema(component, schemas[component_type])
component["meta"]["input_shape"] = validator.get_input_shape(component, components)
component["meta"]["shape"] = validator.get_shape(component, components)
if not lazy:
validator.eval_input_channels(component, components)
validator.validate_connection(component, components)
inputs = component["meta"]["inputs"]
except Exception as err:
msg = "Component %s annotation failed. \n" % (name)
msg += str(err)
logger.error(msg)
logger.error(str(err))
raise Exception(msg)
components[name] = component
return list(components.values())
def validate_connection(self, component, components):
transformations = self.get_transformation()
dims = self.get_dimensions(component, components)
if transformations is None or (dims[1] is None):
return True
else:
valid = False
msg = ""
for trans in transformations:
if valid:
return valid
try:
if len(trans[0]) != len(dims[0]):
_msg = "Number of inputs mismatch %s (ingredient: %s, dim %s)" % (
component["meta"]["name"],
component["meta"]["type"], str(dims))
# logger.error(msg)
raise Exception(msg)
elif len(
list(
filter(lambda x: len(x[0]) != len(x[1]),
zip(trans[0], dims[0])))) != 0:
_msg = "Tensor rank mismatch %s (ingredient: %s, dim: %s)" % (
component["meta"]["name"],
component["meta"]["type"], str(dims))
raise Exception(_msg)
else:
trans_ = [i for sub in trans[0]
for i in sub] + trans[1]
dims_ = [i for sub in dims[0] for i in sub] + dims[1]
n = len(trans_)
pattern_required = [
trans_[i] == trans_[i + j] for i in range(n - 1)
for j in range(1, n - i)
]
pattern_data = [
dims_[i] == dims_[i + j] for i in range(n - 1)
for j in range(1, n - i)
]
for r in pattern_required:
if pattern_required and (not pattern_data):
_msg = "Tensor dimension mismatch" % component[
"meta"]["name"]
raise Exception(_msg)
valid = True
except Exception as err:
msg += "%s\n " % str(err)
# traceback.print_exc()
# raise Exception(msg)
if not valid:
logger.error("Validation failed %s, %s" %
(component["meta"]["name"], msg))
logger.error("Dimension %s" % dims)
raise Exception()
def eval_input_channels(self, component, components):
inputs = self.get_input_shape(component, components)
try:
if inputs is not None:
if inputs[0] is not None:
util.replace_key(component["value"]["hyperparams"],
"input_shape", inputs[0][-1])
except Exception as err:
logger.error(err)
logger.error(
"Ingredient %s (type: %s) invalid inputs" %
(component["meta"]["name"], component["meta"]["type"]))
def connect_to(components, current_component_type, current_component_hyperparams=None, range=None, types=None, scope="all"):
def _get_name_scope(_component):
return "/".join(_component["meta"]["name"].split("/")[:-1])
def _get_type(_component):
return _component["meta"]["type"]
if scope != "all" and (scope != "current"):
logger.error("Scope has to be 'current' or 'all'!")
raise Exception
if current_component_hyperparams is None:
current_component_hyperparams = components[-1]["value"]["hyperparams"]
current_component_name_scope = _get_name_scope(components[-1])
if range is not None:
lower, upper = _parse_range(range)
else:
lower, upper = -2, -1
connected_component = []
if lower == "" and upper != "":
_components = components[:upper]
elif upper == "" and lower != "":
_components = components[lower:]
elif upper != "" and lower != "":
_components = components[lower:upper]
else:
_components = components
for component in _components:
type_match = True
scope_match = True
_component_type = _get_type(component)
if types is not None and (_component_type not in types):
type_match = False
if scope == "current" and (current_component_name_scope != (_get_name_scope(component))):
scope_match = False
if type_match and scope_match:
connected_component.append("%s/%s" % (component["meta"]["name"],
types[_component_type]))
# Make regularization part of the hyperparam subtree
# component["value"]["hyperparams"][types[_component_type]][current_component_type] = current_component_hyperparams
delete_key(component["value"]["hyperparams"], "connect_to")
delete_key(current_component_hyperparams, "connect_to")
return connected_component
def draw_graph(graph_list, level=2, graph_path='example.png', show="name"):
graph = pydot.Dot(graph_type='digraph', rankdir='LR', dpi=800, size=5, fontsize=18)
prev = None
for component in graph_list:
_dir = component[const.META][const.NAME].split("/")
_dir = _dir[:level]
name = "/".join(_dir)
if prev is None:
prev = name
if name == prev:
continue
else:
prev = name
component_scope = component[const.META][const.SCOPE]
component_type = component[const.META][const.TYPE]
node = pydot.Node(name, shape="box")
if show == "name":
label = name.split("/")[-1]# .split("_")[0]
else:
if show not in component["meta"] or show=="type":
logger.error("Lable '%s' is not available" % show)
raise Exception
label = str(component["meta"][show])
node.set_label(label)
graph.add_node(node)
_type = component[const.META][const.TYPE]
if not _type in registry.DATA_BLOCK:
inputs = component[const.META][const.INPUTS]
for _input in inputs:
if _type not in registry.REGULARIZERS:
_input = "/".join(_input.split("/")[:level])
_edge = pydot.Edge(_input, name)
graph.add_edge(_edge)
graph.write_png(graph_path)
return graph
def parse(yml_file, return_dict=False, lazy=True):
global global_count0, global_count1, component_count, prev_component
global_count0 = 0
global_count1 = 0
component_count = 0
# print("parsing, yml_file", yml_file)
prev_component = None
try:
# Read the yml file
if isinstance(yml_file, str):
user_defined = util.read_yaml(yml_file)
elif isinstance(yml_file, dict):
user_defined = deepcopy(yml_file)
else:
msg = "First argument has to be a path or a dict type"
logger.error(msg)
raise TypeError(msg)
reader = alex_reader(user_defined)
# Step 1: transform yml into a (recursive) dict
# Each node in the dict has the following keys:
# hyperparams: list or dict, default: {}
# type: string
# inputs: list, default: None
# name: string, default: None
# (optional) dtype: data type
# (optional) shape: list
_graph:dict = config_to_inter_graph(user_defined, reader)
# Step 2: transform _graph into an intermediate ast, where each key is a function
# name
# inputs
# repeat
# e.g. ("inputs", inputs_list, ("name", name_str, ("repeat", n, [])))
ast:list = inter_graph_to_ast(_graph)
# Step 3: first pass
# A list of dicts
# The reason of having a list of dicts as the main underlying data structure is due to its simplicity. This data structure can be transformed into the final ast
# Each dict has the following structure:
# "meta": {"inputs": None or list,
# "name": string (scope/.../name),
# "scope": the scope of the node,
# "type": type of the deep learning component}
# "value": {"hyperparams": dict of hyperparameters,
# "stats": {}
# "tensor": library specific tensor object (not easily serializable),
# "value": a serializable object that contains the same info as tensor,
# "var": }
graph:list = eval_ast_to_graph(ast)
# Step 4: second pass: same data structure; populate inputs
graph:list = global_update(graph)
# if config_to_type(yml_file) not in const.RECIPE_TYPES:
graph = annotate(graph, lazy=lazy) # json schema
# Step 5: load graph and states from checkpoint; check if the structure matches the one defined in the DSL
except Exception as err:
msg = "Error during parsing configuration %s" % yml_file
msg += "\n %s" % str(err)
logger.error(msg)
raise Exception(str(err))
if return_dict:
graph = list_to_dict(graph)
return graph
def _draw_hyperparam_tree(_graph,
parent_node: str,
graph):
_graph = clone(_graph)
if _is_simple(_graph):
graph = _add_node(parent_node=parent_node,
label=str(_graph),
name="%s/%s" % (parent_node, str(_graph)),
graph=graph)
else:
for i, _node in enumerate(_graph):
""" The assumption here is that _graph can be either a dict
or hyperparams and hyperparams have a very simple
structure, which means that _node is either a string
or a scalar. If _graph is a list and _node is a complex
data structure, it is not handled.
"""
if _is_simple(_node):
label = str(_node) # node is simple
else:
pprint(_node)
logger.error("Unrecognized node type")
raise Exception
name = "%s/%s" % (parent_node, label)
if isinstance(_graph, list):
name = "%s_%i" % (name, i)
if isinstance(_graph, dict) and _is_component(_graph[_node]):
# FIXME:
if not _graph[_node]["meta"]["visible"]:
continue
label = clone(_graph[_node][const.TYPE])
graph = _add_node(parent_node, label, name, graph)
if isinstance(_graph, dict):
_node = clone(_graph[_node])
if _is_simple(_node):
graph = _add_node(parent_node=name,
label=str(_node),
name="%s/%s" % (name, str(_node)),
graph=graph)
elif isinstance(_node, list):
for ii, __node in enumerate(_node):
graph = _add_node(parent_node=name,
label=str(__node),
name="%s/%s_%i" % (name, str(__node), ii),
graph=graph)
if isinstance(_node, dict):
# component or hyperparams
if _is_component(_node): # is a component (ingredient or recipe)
if _is_recipe(_node):
graph = _draw_hyperparam_tree(_node[const.SUBGRAPH],
parent_node=name,
graph=graph)
# go in the hyperparam tree
if recipe_defined(_node[const.TYPE]):
graph = _draw_hyperparam_tree(_node[const.HYPERPARAMS],
parent_node=name,
graph=graph)
else: # in the hyperparam tree
graph = _draw_hyperparam_tree(_node,
parent_node=name,
graph=graph)
return graph