def timer_report(self): local_passed = Time.passed(self.local_timer) global_passed = Time.passed(self.global_timer) self.local_timer = Time.now() return local_passed, global_passed
tokens.append(chr(self.voc[t])) return "".join(tokens) if __name__ == "__main__": g = Graph() g.add_edge('A', 'B', 'b1') # g.add_edge('A', 'B', 'b2') g.add_edge('A', 'C', 'c') g.add_edge('A', 'C', 'c2') # g.add_edge('C', 'C', 'cd') g.add_edge('A', 'D', 'd') g.add_edge('D', 'E', 'e') g.add_edge('A', 'E', 'e') now = Time.now() plans = list(g.exhaustive_plan(force_tree=False).linearizations()) print("exhaustive_plan", len(plans)) print(plans[0]) print(plans[-1]) print(Time.passed(now)) print("memory size", sys.getsizeof(pickle.dumps(plans)) / 1024) plans_str = "\n".join(plans).encode("utf-8") compressed = zlib.compress(plans_str, 1) print("zlib size", sys.getsizeof(pickle.dumps(compressed)) / 1024) compressor = Compressor() plans = [compressor.compress(p) for p in plans] print("compressor size", sys.getsizeof(pickle.dumps({"p": plans, "c": compressor})) / 1024)
def execute(self, run_name=None, tabs=0, x_params=None, previous_name=None, cache_name: str = None): # Every execution should refresh initial params params = CachedDict().union(self.initial_params) \ if isinstance(self.initial_params, CachedDict) \ else CachedDict(self.initial_params if self.initial_params else {}) self.local_timer = Time.now() self.global_timer = Time.now() if not x_params: x_params = CachedDict() x_params = x_params.union(params) # Add F to X if not previous_name: previous_name = cache_dir makedir(previous_name) if cache_name: previous_name = path.join(previous_name, cache_name) makedir(previous_name) if run_name: print(" " * tabs, run_name) key_len = max([len(qi.key) for qi in self.queue] + [0]) + 5 name_len = max([len(qi.name) for qi in self.queue] + [0]) + 5 for qi in self.queue: # key, name, method, load_cache, load_self if qi.key != "out" and not isinstance(qi.method, Pipeline): print((" " * (tabs + 1)) + ("%-" + str(key_len) + "s %-" + str(name_len) + "s") % (qi.key, qi.name), end=" ") pn = path.join(previous_name, qi.key) pnf = pn + "." + qi.ext if qi.load_cache and path.isfile(pnf): params.add_cache(qi.key, pnf) if qi.load_self: params.load_cache(qi.key) else: if isinstance(qi.method, Pipeline): params[qi.key] = qi.method.execute( run_name=qi.name, tabs=tabs + 1, x_params=x_params.union(params), previous_name=pn) if isinstance(params[qi.key], CachedDict) and "out" in params[qi.key]: params.copy_key(qi.key, params[qi.key], "out") else: if self.mute: Silencer.mute() params[qi.key] = qi.method(params, x_params) if self.mute: Silencer.unmute() f = open(pnf, "wb" if qi.ext not in ["txt", "json"] else "w") if qi.ext == "pkl": pickle.dump(params[qi.key], f) else: if qi.ext in ["png", "jpg", "wav", "mp4"]: params[qi.key] = get_file_bytes(params[qi.key], format=qi.ext) f.write(params[qi.key]) f.close() if qi.key != "out" and not isinstance(qi.method, Pipeline): local_passed, global_passed = self.timer_report() report = params[qi.key].report() \ if qi.key in params.val_dict and hasattr(params[qi.key], "report") else "" print(("%-15s\t\t" + report) % (local_passed)) return params
def execute(self, run_name=None, tabs=0, x_params=None, previous_name=None, cache_name: str = None): self.local_timer = Time.now() self.global_timer = Time.now() if not x_params: x_params = CachedDict() if not previous_name: previous_name = cache_dir makedir(previous_name) if cache_name: previous_name = path.join(previous_name, cache_name) makedir(previous_name) if run_name: print(" " * tabs, run_name) key_len = max([len(qi.key) for qi in self.queue] + [0]) + 5 name_len = max([len(qi.name) for qi in self.queue] + [0]) + 5 for qi in self.queue: # key, name, method, load_cache, load_self if qi.key != "out" and not isinstance(qi.method, Pipeline): print((" " * (tabs + 1)) + ("%-" + str(key_len) + "s %-" + str(name_len) + "s") % (qi.key, qi.name), end=" ") pn = path.join(previous_name, qi.key) pnf = pn + "." + qi.ext if qi.load_cache and qi.key != "out" and path.isfile(pnf): self.params.add_cache(qi.key, pnf) if qi.load_self: self.params.load_cache(qi.key) else: if isinstance(qi.method, Pipeline): self.params[qi.key] = qi.method.execute( run_name=qi.name, tabs=tabs + 1, x_params=x_params.union(self.params), previous_name=pn) if "out" in self.params[qi.key]: self.params.copy_key(qi.key, self.params[qi.key], "out") else: if self.mute: Silencer.mute() self.params[qi.key] = qi.method(self.params, x_params) if self.mute: Silencer.unmute() f = open(pnf, "wb" if qi.ext != "txt" else "w") if qi.ext == "sav": pickle.dump(self.params[qi.key], f) else: f.write(self.params[qi.key]) f.close() if qi.key != "out" and not isinstance(qi.method, Pipeline): local_passed, global_passed = self.timer_report() report = self.params[qi.key].report() \ if qi.key in self.params.val_dict and hasattr(self.params[qi.key], "report") else "" print(("%-15s\t\t" + report) % (local_passed)) return self.params