def consolidate_results(glob_path): start = perf_counter() df = load_results_as_frame(glob_path) first_path = glob.glob(glob_path)[0] dir_ = os.path.dirname(first_path) path = f"{dir_}_summary.json" df.to_json(path) shutil.make_archive(dir_, "zip", dir_) shutil.rmtree(dir_) print(utils.elapsed(start))
async def trans_kr(ctx): global now if utils.elapsed(now): try: phrase = ctx.content[ctx.content.index(' ')+1:] print(f'translate {phrase}') response = utils.translate(phrase, 'ko', 'en') print(response) now = time() await ctx.send(f'@{ctx.author.name}, {phrase} = {response.text}') except ValueError: await ctx.send(f'@{ctx.author.name}, usage: !translatekr <text in korean>')
async def trans_detect(ctx): global now if utils.elapsed(now): try: phrase = ctx.content[ctx.content.index(' ')+1:] lang = utils.detect_language(phrase) print(f'translate {phrase}') if lang == 'en': response = utils.translate(phrase, 'en', 'ko') elif lang == 'ko': response = utils.translate(phrase, 'ko', 'en') else: raise Exception('Invalid language.') print(response) now = time() await ctx.send(f'@{ctx.author.name}, {phrase} = {response.text}') except ValueError: await ctx.send(f'@{ctx.author.name}, usage: !translate <text>')
def ols_sweep(data, target, n_vars=2, ignore=None, dst=OLS_SWEEP_DIR, jobs=os.cpu_count()): start = perf_counter() if ignore: data = data.drop(columns=ignore) var_names = utils.noncat_cols(data) var_names += [f"C({x})" for x in utils.cat_cols(data)] var_names.remove(target) combos = list(itertools.combinations(var_names, n_vars)) combo_strs = ["+".join(x) for x in combos] formulae = [f"{target}~{x}" for x in combo_strs] dst = os.path.join(dst, f"{target}~{n_vars}") os.makedirs(dst, exist_ok=True) paths = [os.path.join(dst, f"{x}.json") for x in formulae] with ThreadPool(jobs) as pool: build = partial(_build_and_record, data) pool.starmap(build, zip(formulae, paths)) print(utils.elapsed(start))
def execute(cmd, config="default", experiment_id=None, shutdown=False, debug=False): elapsed() try: run_log = yaml.load(open("checkpoints/runlog.yml")) or {} except: run_log = {} mode = config run_data = run_log[mode] = run_log.get(mode, {}) run_data["runs"] = run_data.get("runs", 0) + 1 run_name = experiment_id or (mode + "_" + str(run_data["runs"])) run_data[run_name] = run_data.get(run_name, {"config": config, "cmd": cmd, "status": "Running"}) run_data = run_data[run_name] print(f"Running job: {run_name}") # shutil.rmtree("output/", ignore_errors=True) # os.makedirs("output/") os.makedirs(f"checkpoints/{run_name}", exist_ok=True) os.system("echo " + run_name + ", 0, mount > scripts/jobinfo.txt") print ("Hello") os.system("echo " + run_name + ", 0, mount > mount/shared/hi.txt") print ("goodbye") print(cmd) cmd = shlex.split(cmd) if cmd[0] == "python" and debug: cmd[0] = "ipython" cmd.insert(1, "-i") elif "python" in cmd[0]: print ("No buffering") cmd.insert(1, "-u") elif "python" in cmd[1]: print ("No buffering") cmd.insert(2, "-u") print(" ".join(cmd)) process = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, universal_newlines=True) try: with open(f"checkpoints/{run_name}/stdout.txt", "w") as outfile: for stdout_line in iter(process.stdout.readline, ""): print(stdout_line, end="") outfile.write(stdout_line) return_code = process.wait() run_data["status"] = "Error" if return_code else "Complete" except KeyboardInterrupt: print("\nKilled by user.") process.kill() run_data["status"] = "Killed" except OSError: print("\nSystem error.") process.kill() run_data["status"] = "Error" process.kill() if debug and run_data["status"] != "Complete": return subprocess.run(["rsync", "-av", "--progress", ".", "checkpoints/" + run_name, "--exclude", "checkpoints", "--exclude", ".git", "--exclude", "data/snapshots", "--exclude", "data/results", "--exclude", "mount"], stdout=subprocess.DEVNULL); yaml.safe_dump(run_log, open("checkpoints/runlog.yml", "w"), allow_unicode=True, default_flow_style=False) yaml.safe_dump(run_data, open(f"checkpoints/{run_name}/comments.yml", "w"), allow_unicode=True, default_flow_style=False) interval = elapsed() print(f"Program ended after {interval:0.4f} seconds.") print ("Shutdown?", shutdown) print (run_data["status"], interval) if shutdown and run_data["status"] != "Killed" and interval > 1: print(f"Shutting down in 5 minutes.") time.sleep(300) subprocess.call("sudo shutdown -h now", shell=True)
def execute(cmd, mode="experiment", config="default", shutdown=False, debug=False): elapsed() try: run_log = yaml.load(open("jobs/runlog.yml")) except: run_log = {} run_data = run_log[mode] = run_log.get(mode, {}) run_data["runs"] = run_data.get("runs", 0) + 1 run_name = mode + str(run_data["runs"]) run_data[run_name] = run_data.get(run_name, { "config": config, "cmd": cmd, "status": "Running" }) run_data = run_data[run_name] print(f"Running job: {run_name}") shutil.rmtree("output/", ignore_errors=True) os.makedirs("output/") os.makedirs(f"jobs/{run_name}", exist_ok=True) with open("jobs/jobinfo.txt", "w") as config_file: print(config, file=config_file) cmd = shlex.split(cmd) if cmd[0] == "python" and debug: cmd[0] = "ipython" cmd.insert(1, "-i") elif cmd[0] == "python": cmd.insert(1, "-u") print(" ".join(cmd)) process = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, universal_newlines=True) try: with open(f"jobs/{run_name}/stdout.txt", "w") as outfile: for stdout_line in iter(process.stdout.readline, ""): print(stdout_line, end="") outfile.write(stdout_line) return_code = process.wait() run_data["status"] = "Error" if return_code else "Complete" except KeyboardInterrupt: print("\nKilled by user.") process.kill() run_data["status"] = "Killed" except OSError: print("\nSystem error.") process.kill() run_data["status"] = "Error" process.kill() if debug and run_data["status"] != "Complete": return shutil.copytree("output", f"jobs/{run_name}/output") for file in glob.glob("*.py"): shutil.copy(file, f"jobs/{run_name}") yaml.safe_dump(run_log, open("jobs/runlog.yml", "w"), allow_unicode=True, default_flow_style=False) yaml.safe_dump(run_data, open(f"jobs/{run_name}/comments.yml", "w"), allow_unicode=True, default_flow_style=False) interval = elapsed() print(f"Program ended after {interval:0.4f} seconds.") if shutdown and run_data["status"] != "Killed" and interval > 60: print(f"Shutting down in 1 minute.") time.sleep(60) subprocess.call("sudo shutdown -h now", shell=True)