def _extract_chain_structure(self, node: Node, operation_id: int,
visited_nodes: List[Node]):
"""
Recursively go through the Chain from 'root_node' to PrimaryNode's,
creating a OperationTemplate with unique id for each Node. In addition,
checking whether this Node has been visited yet.
"""
if node.nodes_from:
nodes_from = []
for node_parent in node.nodes_from:
if node_parent in visited_nodes:
nodes_from.append(visited_nodes.index(node_parent) + 1)
else:
visited_nodes.append(node_parent)
nodes_from.append(len(visited_nodes))
self._extract_chain_structure(node_parent,
len(visited_nodes),
visited_nodes)
else:
nodes_from = []
if node.operation.operation_type == atomized_model_type():
operation_template = AtomizedModelTemplate(node, operation_id,
nodes_from)
else:
operation_template = OperationTemplate(node, operation_id,
nodes_from)
self.operation_templates.append(operation_template)
self.total_chain_operations[operation_template.operation_type] += 1
return operation_template
def _extract_pipeline_structure(self, node: Node, operation_id: int,
visited_nodes: List[Node]):
"""
Recursively go through the Pipeline from 'root_node' to PrimaryNode's,
creating a OperationTemplate with unique id for each Node. In addition,
checking whether this Node has been visited yet.
"""
if node and node.nodes_from:
nodes_from = []
for node_parent in node.nodes_from:
if node_parent in visited_nodes:
nodes_from.append(visited_nodes.index(node_parent) + 1)
else:
visited_nodes.append(node_parent)
nodes_from.append(len(visited_nodes))
self._extract_pipeline_structure(node_parent,
len(visited_nodes),
visited_nodes)
else:
nodes_from = []
# TODO resolve as to GraphTemplate
if hasattr(node, 'operation'):
if (not isinstance(node.operation, str) and
node.operation.operation_type == atomized_model_type()):
operation_template = AtomizedModelTemplate(
node, operation_id, nodes_from)
else:
operation_template = OperationTemplate(node, operation_id,
nodes_from)
self.operation_templates.append(operation_template)
self.total_pipeline_operations[
operation_template.operation_type] += 1
def __init__(self, pipeline: 'Pipeline'):
if not pipeline.root_node:
raise ValueError(
f'AtomizedModel could not create instance of empty Pipeline.')
super().__init__(operation_type=atomized_model_type())
self.pipeline = pipeline
self.unique_id = uuid4()
def __init__(self, chain: 'Chain'):
if not chain.root_node:
raise ValueError(
f'AtomizedModel could not create instance of empty Chain.')
super().__init__(operation_type=atomized_model_type())
self.chain = chain
self.unique_id = uuid4()
def roll_pipeline_structure(
self,
operation_object: ['OperationTemplate', 'AtomizedModelTemplate'],
visited_nodes: dict,
path: str = None):
"""
The function recursively traverses all disjoint operations
and connects the operations in a pipeline.
:params operation_object: operationTemplate or AtomizedOperationTemplate
:params visited_nodes: array to remember which node was visited
:params path: path to save
:return: root_node
"""
if operation_object.operation_id in visited_nodes:
return visited_nodes[operation_object.operation_id]
if operation_object.operation_type == atomized_model_type():
atomized_model = operation_object.next_pipeline_template
if operation_object.nodes_from:
node = SecondaryNode(operation_type=atomized_model)
else:
node = PrimaryNode(operation_type=atomized_model)
else:
if operation_object.nodes_from:
node = SecondaryNode(operation_object.operation_type)
else:
node = PrimaryNode(operation_object.operation_type)
node.operation.params = operation_object.params
node.rating = operation_object.rating
if hasattr(
operation_object, 'fitted_operation_path'
) and operation_object.fitted_operation_path and path is not None:
path_to_operation = os.path.join(
path, operation_object.fitted_operation_path)
if not os.path.isfile(path_to_operation):
message = f"Fitted operation on the path: {path_to_operation} does not exist."
self.log.error(message)
raise FileNotFoundError(message)
fitted_operation = joblib.load(path_to_operation)
operation_object.fitted_operation = fitted_operation
node.fitted_operation = fitted_operation
nodes_from = [
operation_template
for operation_template in self.operation_templates
if operation_template.operation_id in operation_object.nodes_from
]
node.nodes_from = [
self.roll_pipeline_structure(node_from, visited_nodes, path)
for node_from in nodes_from
]
visited_nodes[operation_object.operation_id] = node
return node
def _extract_operations(self, chain_json, path):
operation_objects = chain_json['nodes']
for operation_object in operation_objects:
if operation_object['operation_type'] == atomized_model_type():
filename = operation_object[
'atomized_model_json_path'] + '.json'
curr_path = os.path.join(
os.path.dirname(path),
operation_object['atomized_model_json_path'], filename)
operation_template = AtomizedModelTemplate(path=curr_path)
else:
operation_template = OperationTemplate()
operation_template.import_json(operation_object)
self.operation_templates.append(operation_template)
self.total_chain_operations[operation_template.operation_type] += 1
