def config(self):
if self._cfg is not None:
return self._cfg
if os.path.exists(self.getConfigYamlPath()):
self._cfg = load_yaml(self.getConfigYamlPath())
else:
self._cfg = load_yaml(self.getConfigYamlConcretePath())
return self._cfg
def parse(name: str, p, extra=None):
m = load(name)
if type(p) == str:
with open(p, "r") as f:
first_line = f.readline()
first_line = first_line.strip()
if first_line.startswith("#%Musket "):
first_line = first_line[8:].strip()
dialect = first_line[:first_line.index(' ')].strip()
m = load(dialect.lower())
#dialect=
with open(p, "r") as f:
base = yaml_load(f)
if extra is not None:
extrad = utils.load_yaml(extra)
for v in extrad:
if v not in base:
base[v] = extrad[v]
else:
if v in alllowReplace:
for q in extrad[v]:
mn = base[v]
if mn is not None:
if q not in mn:
mn[q] = extrad[v][q]
else:
base[v] = extrad[v]
break
return m.instantiate(base)
return m.instantiate(p)
def createNet(self):
inp, output = utils.load_yaml(self.path + ".shapes")
if not hasattr(context.context, "net_cx"):
context.context.net_cx = []
contributions = None
if os.path.exists(self.path + ".contribution"):
contributions = utils.load(self.path + ".contribution")
else:
contributions = None
if isinstance(inp, list):
inputs = [self.create_input(x) for x in inp]
if contributions is not None:
if isinstance(contributions, list):
for i in range(len(inputs)):
self.set_contribution(inputs[i], contributions[i])
else:
for i in range(len(inputs)):
self.set_contribution(inputs[i], contributions)
else:
i = keras.Input(inp)
i.contribution = contributions
inputs = [i]
m = net.create_model_from_config(self.declarations, inputs,
self.architecture, self.imports)
if context.isTrainMode():
if hasattr(context.context, "net_cx"):
utils.save(self.path + ".ncx", context.context.net_cx)
context.context.net_cx = []
return m
def modules(self):
res = []
for m in os.listdir(self.modulesPath()):
if m.endswith(".py"):
x = m[:-3]
res.append(self.module(x))
res.append(dataset_analizers)
res.append(dataset_visualizers)
res.append(builtin_datasets)
res.append(metrics)
res.append(crf)
if os.path.exists(self.commonPath()):
mods = utils.load_yaml(self.commonPath())
if mods is not None:
if "imports" in mods:
if isinstance(mods["imports"], list):
for m in mods["imports"]:
res.append(self.module(m))
try:
res.append(self.module("musket_text.preprocessors"))
except:
pass
return res
def create_writeable_dataset(self, dataset: datasets.DataSet,
dsPath: str) -> datasets.WriteableDataSet:
inp, output = utils.load_yaml(self.path + ".shapes")
resName = dataset.get_name() + "_predictions"
result = datasets.BufferedWriteableDS(dataset,
resName,
dsPath,
pickle=self.isMultiOutput())
return result
def readSettings(self, csvPath) -> dict:
if os.path.isabs(csvPath):
absPath = csvPath
else:
absPath = os.path.join(context.get_current_project_data_path(),
csvPath)
fDir = os.path.dirname(absPath)
fName = "." + os.path.basename(absPath) + ".dataset_desc"
settingsPath = os.path.join(fDir, fName)
settingsObj = load_yaml(settingsPath)
return settingsObj
def concrete(self, all=False):
if self.hyperparameters() is not None:
yaml = load_yaml(self.path + "/hyperopt.scores")
max = None
best = None
for v in yaml:
if max is None:
best = v
max = yaml[v]
if yaml[v] > max:
best = v
if best is not None:
concretePath = os.path.join(self.path, "trial" + str(best - 1))
return [Experiment(concretePath)]
return [self]
return self.apply(True)
def createNet(self):
inp,output=utils.load_yaml(self.path + ".shapes")
contributions=None
if os.path.exists(self.path+".contribution"):
contributions=utils.load(self.path+".contribution")
else:
contributions=None
if isinstance(inp,list):
inputs=[keras.Input(x) for x in inp]
if contributions is not None:
for i in range(len(inputs)):
inputs[i].contribution=contributions[i]
else:
i=keras.Input(inp);
i.contribution=contributions
inputs=[i]
m=net.create_model_from_config(self.declarations,inputs,self.architecture,self.imports)
return m
def isMultiOutput(self):
if self._multiOutput is not None:
return self._multiOutput
inp,output=utils.load_yaml(self.path + ".shapes")
self._multiOutput= len(output)>1
return self._multiOutput
def metrics(self):
if os.path.exists(self.getSummaryYamlPath()):
return load_yaml(self.getSummaryYamlPath())
return {}
def isMultiOutput(self):
if self._multiOutput is not None:
return self._multiOutput
inp, output = utils.load_yaml(self.path + ".shapes")
self._multiOutput = len(output) > 1 and isinstance(output, list)
return self._multiOutput
def get_declarations(path) -> keras.Model:
n = load_yaml(path)
d = Declarations(n["declarations"])
return d
def create_model(path, inputs, name="net") -> keras.Model:
n = load_yaml(path)
d = Declarations(n["declarations"])
out = d.model(name, inputs)
return out
