def config(self): if self._cfg is not None: return self._cfg if os.path.exists(self.getConfigYamlPath()): self._cfg = load_yaml(self.getConfigYamlPath()) else: self._cfg = load_yaml(self.getConfigYamlConcretePath()) return self._cfg
def parse(name: str, p, extra=None): m = load(name) if type(p) == str: with open(p, "r") as f: first_line = f.readline() first_line = first_line.strip() if first_line.startswith("#%Musket "): first_line = first_line[8:].strip() dialect = first_line[:first_line.index(' ')].strip() m = load(dialect.lower()) #dialect= with open(p, "r") as f: base = yaml_load(f) if extra is not None: extrad = utils.load_yaml(extra) for v in extrad: if v not in base: base[v] = extrad[v] else: if v in alllowReplace: for q in extrad[v]: mn = base[v] if mn is not None: if q not in mn: mn[q] = extrad[v][q] else: base[v] = extrad[v] break return m.instantiate(base) return m.instantiate(p)
def createNet(self): inp, output = utils.load_yaml(self.path + ".shapes") if not hasattr(context.context, "net_cx"): context.context.net_cx = [] contributions = None if os.path.exists(self.path + ".contribution"): contributions = utils.load(self.path + ".contribution") else: contributions = None if isinstance(inp, list): inputs = [self.create_input(x) for x in inp] if contributions is not None: if isinstance(contributions, list): for i in range(len(inputs)): self.set_contribution(inputs[i], contributions[i]) else: for i in range(len(inputs)): self.set_contribution(inputs[i], contributions) else: i = keras.Input(inp) i.contribution = contributions inputs = [i] m = net.create_model_from_config(self.declarations, inputs, self.architecture, self.imports) if context.isTrainMode(): if hasattr(context.context, "net_cx"): utils.save(self.path + ".ncx", context.context.net_cx) context.context.net_cx = [] return m
def modules(self): res = [] for m in os.listdir(self.modulesPath()): if m.endswith(".py"): x = m[:-3] res.append(self.module(x)) res.append(dataset_analizers) res.append(dataset_visualizers) res.append(builtin_datasets) res.append(metrics) res.append(crf) if os.path.exists(self.commonPath()): mods = utils.load_yaml(self.commonPath()) if mods is not None: if "imports" in mods: if isinstance(mods["imports"], list): for m in mods["imports"]: res.append(self.module(m)) try: res.append(self.module("musket_text.preprocessors")) except: pass return res
def create_writeable_dataset(self, dataset: datasets.DataSet, dsPath: str) -> datasets.WriteableDataSet: inp, output = utils.load_yaml(self.path + ".shapes") resName = dataset.get_name() + "_predictions" result = datasets.BufferedWriteableDS(dataset, resName, dsPath, pickle=self.isMultiOutput()) return result
def readSettings(self, csvPath) -> dict: if os.path.isabs(csvPath): absPath = csvPath else: absPath = os.path.join(context.get_current_project_data_path(), csvPath) fDir = os.path.dirname(absPath) fName = "." + os.path.basename(absPath) + ".dataset_desc" settingsPath = os.path.join(fDir, fName) settingsObj = load_yaml(settingsPath) return settingsObj
def concrete(self, all=False): if self.hyperparameters() is not None: yaml = load_yaml(self.path + "/hyperopt.scores") max = None best = None for v in yaml: if max is None: best = v max = yaml[v] if yaml[v] > max: best = v if best is not None: concretePath = os.path.join(self.path, "trial" + str(best - 1)) return [Experiment(concretePath)] return [self] return self.apply(True)
def createNet(self): inp,output=utils.load_yaml(self.path + ".shapes") contributions=None if os.path.exists(self.path+".contribution"): contributions=utils.load(self.path+".contribution") else: contributions=None if isinstance(inp,list): inputs=[keras.Input(x) for x in inp] if contributions is not None: for i in range(len(inputs)): inputs[i].contribution=contributions[i] else: i=keras.Input(inp); i.contribution=contributions inputs=[i] m=net.create_model_from_config(self.declarations,inputs,self.architecture,self.imports) return m
def isMultiOutput(self): if self._multiOutput is not None: return self._multiOutput inp,output=utils.load_yaml(self.path + ".shapes") self._multiOutput= len(output)>1 return self._multiOutput
def metrics(self): if os.path.exists(self.getSummaryYamlPath()): return load_yaml(self.getSummaryYamlPath()) return {}
def isMultiOutput(self): if self._multiOutput is not None: return self._multiOutput inp, output = utils.load_yaml(self.path + ".shapes") self._multiOutput = len(output) > 1 and isinstance(output, list) return self._multiOutput
def get_declarations(path) -> keras.Model: n = load_yaml(path) d = Declarations(n["declarations"]) return d
def create_model(path, inputs, name="net") -> keras.Model: n = load_yaml(path) d = Declarations(n["declarations"]) out = d.model(name, inputs) return out