def _ens_pipeline(self):
ens_pl = Pipeline(ensemble=self)
ens_pl.planner_result = self.train_result
ens_pl.finished = True
return ens_pl
def _create_subpipelines(self, primitive, prim_requirements):
mainlst = []
requirement_permutations = list(
itertools.permutations(prim_requirements))
for requirements in requirement_permutations:
#print("%s requirement: %s" % (primitive.name, requirements))
xpipe = Pipeline()
lst = [xpipe]
# Fulfill all requirements of the primitive
for requirement in requirements:
reqvalue = prim_requirements[requirement]
glues = self.glues.getPrimitivesByEffect(requirement, reqvalue)
if len(glues) == 1:
prim = glues[0]
#print("-> Adding one %s" % prim.name)
xpipe.insertPrimitiveAt(0, prim)
elif len(glues) > 1:
newlst = []
for pipe in lst:
# lst.remove(pipe)
for prim in glues:
cpipe = pipe.clone()
#print("-> Adding %s" % prim.name)
cpipe.insertPrimitiveAt(0, prim)
newlst.append(cpipe)
lst = newlst
mainlst += lst
return mainlst
def l1_to_proxy_pipeline(self, l1_pipeline):
pipeline = Pipeline()
ok = True
for prim in l1_pipeline.get_primitives():
l2prim = self.primitive_hash.get(prim.name, None)
if not l2prim:
ok = False
break
pipeline.addPrimitive(l2prim)
if ok:
return pipeline
return None
def find_similar_learner(self, pipeline : Pipeline, include_siblings=True,
num_pipelines=5, position=-1) -> typing.List[Pipeline]:
'''Use ontology to find similar learners'''
# Assume learner is last primitive in pipeline
if position < 0:
position = pipeline.length() - 1
learner = pipeline.getPrimitiveAt(position)
# primitives under the same subtree are similar
nodes = [self.ontology.hierarchy.get_node_by_primitive(learner)]
# primitives under sibiling subtrees are similar
if include_siblings:
nodes = nodes + nodes[0].get_siblings()
# Get primitives but remove the learner itself
primitives_list = [self.ontology.hierarchy.get_primitives_as_list(n) for n in nodes]
if learner in primitives_list[0]:
primitives_list[0].remove(learner)
# Weight primitives under the same subtree more
factor = 10
weights = [factor*p.weight for p in primitives_list[0]]
for primitives in primitives_list[1:]:
weights = weights + [p.weight for p in primitives]
selected = random_choices_without_replacement(
[p for primitives in primitives_list for p in primitives],
weights, num_pipelines)
new_pipelines = []
for p in selected:
# Set task type for execute_pipeline
p.task = 'Modeling'
if not self._check_primitive_include(p):
continue
new_pipeline = pipeline.clone()
new_pipeline.replacePrimitiveAt(position, p)
new_pipelines.append(new_pipeline)
return new_pipelines
def fill_feature_by_weights(self, pipeline : Pipeline, num_pipelines=5) -> Pipeline:
"""Insert feature primitive weighted by on media_type"""
selected_primitives = []
feature_primitive_paths = self.primitive_family_mappings.get_primitives_by_media(
self.media_type)
feature_primitives = []
for path in feature_primitive_paths:
if self.primitive_library.has_primitive_by_package(path):
feature_primitives.append(self.primitive_library.get_primitive_by_package(path))
else:
new_primitive = self.primitive_library.add_custom_primitive(path)
if new_primitive:
print('Adding custom primitive to library: {}'.format(path))
feature_primitives.append(new_primitive)
else:
print('Library does not have primitive {}'.format(path))
print('Possible error in file primitive_family_mappings.json')
primitive_weights = [p.weight for p in feature_primitives]
if feature_primitives:
selected_primitives.extend(random_choices_without_replacement
(feature_primitives, primitive_weights, num_pipelines))
new_pipelines = []
for p in selected_primitives:
# Set task type for execute_pipeline
p.task = 'FeatureExtraction'
newp = pipeline.clone()
newp.insertPrimitiveAt(0, p)
new_pipelines.append(newp)
if new_pipelines:
return new_pipelines
else:
return [pipeline.clone()]
def generate_new_pipelines(self, pipeline: Pipeline, num_pipelines=1):
new_pipelines = []
for i in range(num_pipelines):
new_pipe = pipeline.clone()
new_pipe.planner_result = None
new_pipe.test_result = None
new_pipe.finished = False
for primitive in new_pipe.primitives:
if primitive.task == "Modeling" and primitive.hasHyperparamClass(
):
hyperparams_class = primitive.getHyperparamClass()
primitive.setHyperparams(hyperparams_class.sample())
new_pipelines.append(new_pipe)
print('RandomHyperparamTuning: {}'.format(new_pipelines))
return new_pipelines
def generate_pipelines_with_hierarchy(self, level=2) -> typing.List[Pipeline]:
# ignore level for now, since only have one level hierarchy
results = []
families = self.primitive_family_mappings.get_families_by_task_type(self.task_type.value)
if self.task_type == TaskType.GRAPH_MATCHING:
# if GRAPH_MATCHING then add LINK_PREDICTION
families = families + self.primitive_family_mappings.get_families_by_task_type(TaskType.LINK_PREDICTION)
# moved to get_child_nodes
#families = sebt(families)-set(self.exclude_families
family_nodes = [self.ontology.get_family(f) for f in families]
# include / exclude family and type checks
child_nodes = self._get_child_nodes(family_nodes)
for node in child_nodes:
primitives = self.ontology.hierarchy.get_primitives_as_list(node)
if not primitives:
continue
# Inclusion checks
primitives = [p for p in primitives if self._check_primitive_include(p)]
if not primitives:
continue
weights = [p.weight for p in primitives]
# Set task type for execute_pipeline
for p in primitives:
p.task = 'Modeling'
primitive = random_choices(primitives, weights)
pipe = Pipeline(primitives=[primitive])
results.append(pipe)
return results