def create_default_map(self, template):
default_map = {}
ordered_args = hdfgutils.get_attribute_value(
template.attributes['ordered_args'])
for a in ordered_args:
if a not in list(template.edge_info):
logging.error(
f"Argument {a} not found in edges for {template.op_type}")
edge = template.edge_info[a]
if 'default' in list(edge.attributes):
dtype = hdfgutils.get_attribute_value(edge.attributes['type'])
default_map[a] = CMLANG_CAST_MAP[dtype](
hdfgutils.get_attribute_value(edge.attributes['default']))
else:
default_map[a] = None
return default_map
def create_op_args(self, op_name, node, node_signature, scope):
op_config = self.translator_config['ops'][op_name]
instance_args = hdfgutils.get_attribute_value(
node.attributes['ordered_args'])
signature_args = hdfgutils.get_attribute_value(
node_signature.attributes['ordered_args'])
default_map = self.create_default_map(self.templates[op_name])
for i in range(len(instance_args)):
instance_args[i] = scope + instance_args[i]
args = self.get_ordered_args(op_config, signature_args, instance_args,
default_map, op_name, scope)
kwargs = self.get_kwargs(op_config, signature_args, instance_args,
default_map, op_name, scope)
output_keys = self.get_output_keys(op_config, signature_args,
instance_args, op_name, scope)
return args, kwargs, output_keys
def GetPydotGraph(
graph,
name=None,
inputs=None,
include_state=False,
rankdir='LR',
op_node_producer=None,
blob_node_producer=None,
pairs=[]
):
if op_node_producer is None:
op_node_producer = GetOpNodeProducer(True, **OP_STYLE)
if blob_node_producer is None:
blob_node_producer = GetBlobNodeProducer(**QUEUE_STYLE)
nodes = graph.sub_graph
pydot_graph = pydot.Cluster(name, label=graph.op_type, **CLUSTER_STYLE)
pydot_node_counts = defaultdict(int)
for n in nodes:
op_cat = n.op_cat
if op_cat == 'argument':
continue
elif op_cat == 'component':
node_args = hdfgutils.get_attribute_value(n.attributes['ordered_args'])
pysubgraph = GetPydotGraph(n, n.name,inputs=node_args,rankdir=rankdir)
pydot_graph.add_subgraph(pysubgraph)
else:
op_node = op_node_producer(n, n.gid)
pydot_graph.add_node(op_node)
for i in n.input:
input_name = graph.edge_info[i].name
if input_name not in edge_ids.keys():
input_node = blob_node_producer(
_escape_label(
'edge' + str(graph.edge_info[i].gid)),
label=_escape_label(input_name),
)
edge_ids[input_name] = input_node
else:
input_node = edge_ids[input_name]
pydot_graph.add_node(input_node)
eid_hash = '(' + input_node.get_name() + "," + op_node.get_name() + ')'
backward_hash = '(' + op_node.get_name() + "," + input_node.get_name() + ')'
if eid_hash not in pairs:
pydot_graph.add_edge(pydot.Edge(input_node, op_node))
pairs.append(eid_hash)
pairs.append(backward_hash)
for o in n.output:
output_name = graph.edge_info[o].name
if output_name in edge_ids:
# we are overwriting an existing blob. need to updat the count.
pydot_node_counts[output_name] += 1
output_node = blob_node_producer(
_escape_label(
'edge' + str(graph.edge_info[o].gid)),
label=_escape_label(output_name),
)
edge_ids[output_name] = output_node
pydot_graph.add_node(output_node)
eid_hash = '(' + op_node.get_name() + "," + output_node.get_name() + ')'
backward_hash = '(' + output_node.get_name() + "," + op_node.get_name() + ')'
if eid_hash not in pairs:
pydot_graph.add_edge(pydot.Edge(op_node, output_node))
pairs.append(eid_hash)
pairs.append(backward_hash)
if graph.edge_info[o].iid and graph.edge_info[o].vid:
vid = graph.edge_info[o].vid
if vid in graph.output:
name = graph.edge_info[vid].name
vid_node = edge_ids[name]
eid_hash = '(' + output_node.get_name() + "," + vid_node.get_name() + ')'
backward_hash = '(' + vid_node.get_name() + "," + output_node.get_name() + ')'
if eid_hash not in pairs:
pydot_graph.add_edge(pydot.Edge(output_node, vid_node))
pairs.append(eid_hash)
pairs.append(backward_hash)
if include_state:
for s in n.state:
state_name = graph.edge_info[s].name
if state_name in edge_ids:
# we are overwriting an existing blob. need to updat the count.
pydot_node_counts[state_name] += 1
state_node = blob_node_producer(
_escape_label(
'edge' + str(graph.edge_info[s].gid)),
label=_escape_label(state_name),
)
edge_ids[state_name] = state_node
pydot_graph.add_node(state_node)
eid_hash = '(' + op_node.get_name() + "," + state_node.get_name() + ')'
backward_hash = '(' + state_node.get_name() + "," + op_node.get_name() + ')'
if eid_hash not in pairs:
pydot_graph.add_edge(pydot.Edge(op_node, state_node))
pydot_graph.add_edge(pydot.Edge(state_node, op_node))
pairs.append(eid_hash)
pairs.append(backward_hash)
if graph.edge_info[s].iid and graph.edge_info[s].vid:
vid = graph.edge_info[s].vid
if vid in graph.output:
name = graph.edge_info[vid].name
vid_node = edge_ids[name]
eid_hash = '(' + state_node.get_name() + "," + vid_node.get_name() + ')'
backward_hash = '(' + vid_node.get_name() + "," + state_node.get_name() + ')'
if eid_hash not in pairs:
pydot_graph.add_edge(pydot.Edge(state_node, vid_node))
pairs.append(eid_hash)
pairs.append(backward_hash)
return pydot_graph
def get_arg_attribute(self, key, instance_name, literal=False):
if isinstance(key, list):
arg = []
for k in key:
arg.append(
self.get_arg_attribute(k, instance_name, literal=literal))
return arg
elif isinstance(key, dict):
assert 'func_name' in key.keys() and 'func_args' in key.keys()
return get_func(key['func_name'])(self.get_arg_attribute(
key['func_args'], instance_name))
elif isinstance(key, str):
if key == 'name':
return instance_name
elif key == 'input_op':
return self.op_name
elif key == 'shape':
if literal:
logging.error(
f"Cannot get shape for literal value {instance_name} as attribute"
)
exit(1)
edge = self.translated_graph.edge_info[instance_name]
if 'dimensions' not in list(edge.attributes):
logging.error(f"No dimensions for edge {instance_name}")
tuple_dims = ()
else:
dimensions = hdfgutils.get_attribute_value(
edge.attributes['dimensions'])
tuple_dims = tuple(
int(d) if is_number(d) else d for d in dimensions)
return tuple_dims
elif key == 'type':
if literal:
return type(instance_name).__name__
edge = self.translated_graph.edge_info[instance_name]
if 'type' not in list(edge.attributes):
logging.error(f"No type for edge {instance_name}")
dtype = 'float32'
else:
dtype = hdfgutils.get_attribute_value(
edge.attributes['type'])
return dtype
elif key == "graph":
return self.target_graph
elif key == "scope":
return self.scope
elif key == "relative_scope":
if len(self.scope.split("/")) > 1:
return self.scope.split("/")[-2]
else:
return ''
elif instance_name in self.graph_variables.keys():
return self.graph_variables[instance_name]
else:
return key
elif isinstance(key, bool) or isinstance(key, int):
return key
else:
logging.error(
f"Could not evaluate argument {key} for {instance_name}")
def init_var(self, var, instance_name, literal=False):
args = []
kwargs = {}
arg_type = var['arg_type']
if isinstance(instance_name, str):
id = instance_name.rsplit('/', 1)
if len(id) > 1:
scope = id[0]
id = id[-1]
else:
scope = 'main'
id = id[0]
else:
id = str(instance_name).rsplit('/', 1)
if len(id) > 1:
scope = id[0]
id = id[-1]
else:
scope = 'main'
id = id[0]
if arg_type == 'parameter' and not literal and not is_literal(id):
if instance_name not in list(self.translated_graph.edge_info):
logging.error(
f"Unable to get value for parameter {instance_name}")
exit(1)
edge = self.translated_graph.edge_info[instance_name]
if 'value' not in list(edge.attributes):
logging.error(
f"Could not find literal for parameter argument {instance_name}.\n"
f"Possible attributes: {list(edge.attributes)}")
exit(1)
value = hdfgutils.get_attribute_value(edge.attributes['value'])
elif is_literal(id) and isinstance(instance_name, str):
if id in list(self.translated_graph.edge_info):
edge = self.translated_graph.edge_info[id]
value = hdfgutils.get_attribute_value(edge.attributes['value'])
elif instance_name in list(self.translated_graph.edge_info):
edge = self.translated_graph.edge_info[instance_name]
value = hdfgutils.get_attribute_value(edge.attributes['value'])
else:
logging.error(
f"Could not find literal for parameter argument {instance_name} with id {id}.\n"
f"var: {var['key']}")
exit(1)
else:
value = instance_name
for a in var['init_func_args']:
arg_result = self.get_arg_attribute(a, value, literal=literal)
args.append(arg_result)
for k in var['init_func_kw'].keys():
kwargs[k] = self.get_arg_attribute(var['init_func_kw'][k],
value,
literal=literal)
if len(kwargs.keys()) == 0:
var = get_func(var['init_func'])(*args)
else:
var = get_func(var['init_func'])(*args, **kwargs)
return var
def hdfg_proto(graph, template_name, scope, inst_args):
graph.name = scope
edge_info = templates[template_name].edge_info
edges = get_attribute_value(program.attributes['edges'])
components = get_attribute_value(program.attributes['components'])
t_inputs = templates[template_name].input
t_outputs = templates[template_name].output
t_states = templates[template_name].state
t_params = templates[template_name].parameters
args = get_attribute_value(
templates[template_name].attributes['ordered_args'])
for e in edge_info:
if e in args:
idx = args.index(e)
if (len(inst_args) - 1) < idx:
edge_name = scope + sep + e
graph.edge_info[e].CopyFrom(edge_info[e])
graph.edge_info[e].name = edge_name
graph.edge_info[e].gid = edges
edges += 1
else:
graph.edge_info[e].CopyFrom(inst_args[idx])
else:
edge_name = scope + sep + e
graph.edge_info[e].CopyFrom(edge_info[e])
graph.edge_info[e].name = edge_name
graph.edge_info[e].gid = edges
edges += 1
# graph.input.extend(t_inputs)
# graph.output.extend(t_outputs)
# graph.state.extend(t_states)
# graph.parameters.extend(t_params)
edge_attr = make_attribute('edges', edges)
program.attributes['edges'].CopyFrom(edge_attr)
for n in templates[template_name].sub_graph:
node_type = n.op_cat
new_node = Component()
new_name = scope + sep + n.name
components += 1
if node_type == 'component':
component_attr = make_attribute('components', components)
program.attributes['components'].CopyFrom(component_attr)
node_arg_names = get_attribute_value(
n.attributes['ordered_args'])
node_args = []
for node_arg in node_arg_names:
node_args.append(graph.edge_info[node_arg])
hdfg_proto(new_node, n.op_type, new_name, node_args)
components = get_attribute_value(
program.attributes['components'])
for key in n.attributes:
new_node.attributes[key].CopyFrom(n.attributes[key])
new_node.input.extend(n.input)
new_node.output.extend(n.output)
new_node.state.extend(n.state)
new_node.parameters.extend(n.parameters)
new_node.op_type = n.op_type
new_node.op_cat = n.op_cat
else:
new_node.CopyFrom(n)
new_node.name = new_name
new_node.gid = components - 1
graph.sub_graph.extend([new_node])
component_attr = make_attribute('components', components)
program.attributes['components'].CopyFrom(component_attr)
def flatten_graph(output_graph, graph, templates, context, edge_node_ids,
arg_map):
components = {}
for e in graph.edge_info:
copy_edge = ValueInfo()
if is_literal(e):
uid = str(e)
copy_edge.CopyFrom(graph.edge_info[e])
copy_edge.name = uid
elif e in arg_map.keys():
uid = context + e
copy_edge.CopyFrom(arg_map[e])
copy_edge.attributes['alias'].CopyFrom(
hdfgutils.make_attribute('alias', arg_map[e].name))
else:
uid = context + e
copy_edge.CopyFrom(graph.edge_info[e])
copy_edge.name = uid
if e in graph.input and e not in output_graph.input:
output_graph.input.extend([uid])
elif e in graph.state and e not in output_graph.state:
output_graph.state.extend([uid])
elif e in graph.output and e not in output_graph.output:
output_graph.output.extend([uid])
elif e in graph.parameters and e not in output_graph.parameters:
output_graph.parameters.extend([uid])
if graph.name != 'main':
ordered_args = hdfgutils.get_attribute_value(
graph.attributes['ordered_args'])
else:
ordered_args = []
if 'dimensions' in list(copy_edge.attributes):
dims = hdfgutils.get_attribute_value(
copy_edge.attributes['dimensions'])
new_dims = []
for d in dims:
if d in arg_map.keys():
new_dims.append(arg_map[d].name)
else:
new_dims.append(d)
copy_edge.attributes['dimensions'].CopyFrom(
hdfgutils.make_attribute('dimensions', new_dims))
if uid not in edge_node_ids['edges'].keys():
edge_node_ids['edges'][uid] = str(
len(edge_node_ids['edges'].keys()))
output_graph.edge_info[uid].CopyFrom(copy_edge)
if e not in arg_map.keys():
output_graph.edge_info[uid].gid = int(
edge_node_ids['edges'][uid])
output_graph.edge_info[uid].attributes[
'component_type'].CopyFrom(
hdfgutils.make_attribute('component_type',
graph.op_type))
for n in graph.sub_graph:
op_cat = n.op_cat
if op_cat == 'component':
if n.op_type in components.keys():
components[n.op_type] += 1
new_context = context + n.op_type + str(
components[n.op_type]) + '/'
else:
components[n.op_type] = 0
new_context = context + n.op_type + str(
components[n.op_type]) + '/'
instance_args = hdfgutils.get_attribute_value(
n.attributes['ordered_args'])
ctemplate = templates[n.op_type]
signature_args = hdfgutils.get_attribute_value(
ctemplate.attributes['ordered_args'])
carg_map = create_map(instance_args, signature_args,
graph.edge_info, ctemplate.edge_info,
templates[n.op_type])
update_statement_graphs(ctemplate, output_graph, new_context)
flatten_graph(output_graph, ctemplate, templates, new_context,
edge_node_ids, carg_map)
else:
new = update_node(n, context, arg_map)
if new.name not in edge_node_ids['nodes'].keys():
edge_node_ids['nodes'][new.name] = str(
len(edge_node_ids['nodes'].keys()))
new.gid = int(edge_node_ids['nodes'][new.name])
output_graph.sub_graph.extend([new])
def create_map(instance_args,
signature_args,
instance_edges,
signature_edges,
op=None):
carg_map = {}
for i in range(len(instance_args)):
iarg = instance_args[i]
sarg = signature_args[i]
if is_number(iarg):
iarg = str(iarg)
carg_map[sarg] = instance_edges[iarg]
carg_map[sarg].name = iarg
idims = hdfgutils.get_attribute_value(
instance_edges[iarg].attributes['dimensions'])
iid_literal = False
if instance_edges[iarg].iid:
inst_iid = instance_edges[iarg].iid
iid_literal = is_literal(inst_iid)
sdims = hdfgutils.get_attribute_value(
signature_edges[sarg].attributes['dimensions'])
if len(idims) != len(sdims) and not iid_literal:
logging.error(
"Error! Dimensions between edges connecting components do not match:{} versus {} for {} and {}"
.format(idims, sdims, iarg, sarg))
elif not iid_literal:
for d in range(len(idims)):
inst_dim = idims[d]
sig_dim = sdims[d]
if is_number(inst_dim):
inst_dim = str(inst_dim)
carg_map[sig_dim] = instance_edges[inst_dim]
carg_map[sig_dim].name = inst_dim
carg_map[sig_dim].attributes['vtype'].CopyFrom(
hdfgutils.make_attribute('vtype', 'scalar'))
if len(signature_args) > len(instance_args):
start = len(instance_args)
for default in signature_args[start:]:
sig_attr = list(signature_edges[default].attributes)
if 'default' not in sig_attr:
logging.error(
"Error! No default value for unspecified arg: {}".format(
default))
else:
def_val = hdfgutils.get_attribute_value(
signature_edges[default].attributes['default'])
carg_map[default] = signature_edges[default]
carg_map[default].attributes['value'].CopyFrom(
hdfgutils.make_attribute('value', def_val))
if is_number(def_val):
def_val = str(def_val)
carg_map[default].name = def_val
carg_map[default].attributes['vtype'].CopyFrom(
hdfgutils.make_attribute('vtype', 'scalar'))
for e in op.edge_info:
vcat = hdfgutils.get_attribute_value(
op.edge_info[e].attributes['vcat'])
if vcat == 'declaration':
dims = hdfgutils.get_attribute_value(
op.edge_info[e].attributes['dimensions'])
sig_name = op.edge_info[e].name.rsplit("/", 1)[-1]
return carg_map