def timer_report(self):
local_passed = Time.passed(self.local_timer)
global_passed = Time.passed(self.global_timer)
self.local_timer = Time.now()
return local_passed, global_passed
tokens.append(chr(self.voc[t]))
return "".join(tokens)
if __name__ == "__main__":
g = Graph()
g.add_edge('A', 'B', 'b1')
# g.add_edge('A', 'B', 'b2')
g.add_edge('A', 'C', 'c')
g.add_edge('A', 'C', 'c2')
# g.add_edge('C', 'C', 'cd')
g.add_edge('A', 'D', 'd')
g.add_edge('D', 'E', 'e')
g.add_edge('A', 'E', 'e')
now = Time.now()
plans = list(g.exhaustive_plan(force_tree=False).linearizations())
print("exhaustive_plan", len(plans))
print(plans[0])
print(plans[-1])
print(Time.passed(now))
print("memory size", sys.getsizeof(pickle.dumps(plans)) / 1024)
plans_str = "\n".join(plans).encode("utf-8")
compressed = zlib.compress(plans_str, 1)
print("zlib size", sys.getsizeof(pickle.dumps(compressed)) / 1024)
compressor = Compressor()
plans = [compressor.compress(p) for p in plans]
print("compressor size", sys.getsizeof(pickle.dumps({"p": plans, "c": compressor})) / 1024)
def execute(self,
run_name=None,
tabs=0,
x_params=None,
previous_name=None,
cache_name: str = None):
# Every execution should refresh initial params
params = CachedDict().union(self.initial_params) \
if isinstance(self.initial_params, CachedDict) \
else CachedDict(self.initial_params if self.initial_params else {})
self.local_timer = Time.now()
self.global_timer = Time.now()
if not x_params:
x_params = CachedDict()
x_params = x_params.union(params) # Add F to X
if not previous_name:
previous_name = cache_dir
makedir(previous_name)
if cache_name:
previous_name = path.join(previous_name, cache_name)
makedir(previous_name)
if run_name:
print(" " * tabs, run_name)
key_len = max([len(qi.key) for qi in self.queue] + [0]) + 5
name_len = max([len(qi.name) for qi in self.queue] + [0]) + 5
for qi in self.queue:
# key, name, method, load_cache, load_self
if qi.key != "out" and not isinstance(qi.method, Pipeline):
print((" " * (tabs + 1)) +
("%-" + str(key_len) + "s %-" + str(name_len) + "s") %
(qi.key, qi.name),
end=" ")
pn = path.join(previous_name, qi.key)
pnf = pn + "." + qi.ext
if qi.load_cache and path.isfile(pnf):
params.add_cache(qi.key, pnf)
if qi.load_self:
params.load_cache(qi.key)
else:
if isinstance(qi.method, Pipeline):
params[qi.key] = qi.method.execute(
run_name=qi.name,
tabs=tabs + 1,
x_params=x_params.union(params),
previous_name=pn)
if isinstance(params[qi.key],
CachedDict) and "out" in params[qi.key]:
params.copy_key(qi.key, params[qi.key], "out")
else:
if self.mute:
Silencer.mute()
params[qi.key] = qi.method(params, x_params)
if self.mute:
Silencer.unmute()
f = open(pnf,
"wb" if qi.ext not in ["txt", "json"] else "w")
if qi.ext == "pkl":
pickle.dump(params[qi.key], f)
else:
if qi.ext in ["png", "jpg", "wav", "mp4"]:
params[qi.key] = get_file_bytes(params[qi.key],
format=qi.ext)
f.write(params[qi.key])
f.close()
if qi.key != "out" and not isinstance(qi.method, Pipeline):
local_passed, global_passed = self.timer_report()
report = params[qi.key].report() \
if qi.key in params.val_dict and hasattr(params[qi.key], "report") else ""
print(("%-15s\t\t" + report) % (local_passed))
return params
def execute(self,
run_name=None,
tabs=0,
x_params=None,
previous_name=None,
cache_name: str = None):
self.local_timer = Time.now()
self.global_timer = Time.now()
if not x_params:
x_params = CachedDict()
if not previous_name:
previous_name = cache_dir
makedir(previous_name)
if cache_name:
previous_name = path.join(previous_name, cache_name)
makedir(previous_name)
if run_name:
print(" " * tabs, run_name)
key_len = max([len(qi.key) for qi in self.queue] + [0]) + 5
name_len = max([len(qi.name) for qi in self.queue] + [0]) + 5
for qi in self.queue:
# key, name, method, load_cache, load_self
if qi.key != "out" and not isinstance(qi.method, Pipeline):
print((" " * (tabs + 1)) +
("%-" + str(key_len) + "s %-" + str(name_len) + "s") %
(qi.key, qi.name),
end=" ")
pn = path.join(previous_name, qi.key)
pnf = pn + "." + qi.ext
if qi.load_cache and qi.key != "out" and path.isfile(pnf):
self.params.add_cache(qi.key, pnf)
if qi.load_self:
self.params.load_cache(qi.key)
else:
if isinstance(qi.method, Pipeline):
self.params[qi.key] = qi.method.execute(
run_name=qi.name,
tabs=tabs + 1,
x_params=x_params.union(self.params),
previous_name=pn)
if "out" in self.params[qi.key]:
self.params.copy_key(qi.key, self.params[qi.key],
"out")
else:
if self.mute:
Silencer.mute()
self.params[qi.key] = qi.method(self.params, x_params)
if self.mute:
Silencer.unmute()
f = open(pnf, "wb" if qi.ext != "txt" else "w")
if qi.ext == "sav":
pickle.dump(self.params[qi.key], f)
else:
f.write(self.params[qi.key])
f.close()
if qi.key != "out" and not isinstance(qi.method, Pipeline):
local_passed, global_passed = self.timer_report()
report = self.params[qi.key].report() \
if qi.key in self.params.val_dict and hasattr(self.params[qi.key], "report") else ""
print(("%-15s\t\t" + report) % (local_passed))
return self.params