def create_shortcut(config_path, icon_url=None): icon_url = ( icon_url or "https://icons.iconarchive.com/icons/vexels/office/512/desktop-icon.png" ) deskr_path = os.path.abspath(__file__) absolute_config_path = os.path.abspath(config_path) name = os.path.basename(config_path).replace(".deskr.yaml", "") desktop_folder = os.path.abspath( os.path.join(os.path.expanduser("~"), ".local/share/applications")) downloaded_icon_path = f"{desktop_folder}/{name}_icon.png" sh(f"wget -qO- {icon_url} > {downloaded_icon_path}") desktop_contents = f"""#!/usr/bin/env xdg-open [Desktop Entry] Version=1.0 Terminal=false Type=Application Name={name} Exec=/usr/bin/env python3 {deskr_path} layout {absolute_config_path} Icon={downloaded_icon_path} """ desktop_path = os.path.join(desktop_folder, name + ".desktop") with open(desktop_path, "w") as desktop_file: print(f"Writing {desktop_path}") desktop_file.write(desktop_contents) sh(f"sudo chmod +x {desktop_path}", print_command=True)
def execute(command, *args, **kwargs): logger.debug("Executing: {}".format(command)) result = sh(command, *args, **kwargs) if result['rc'] != SUCCESS: raise Exception("Error executing `{}`. result=`{}`, status=`{}`".format(command, result['rc'], result['status'])) return result
def run_jd(jds, output_dir, jeeves_kwargs): result = { "rc": ERROR_EXCEPTION, "status": "", "jd": jds } time.sleep(random.random() * MAX_DELAY_BEFORE_JEEVES_START) logger.info("JOB_ID: %d" % jds['job_id']) try: with tempfile.NamedTemporaryFile() as fh: json.dump(jds, fh, indent=2, sort_keys=True) fh.flush() runner = os.path.join(my_dir(), "jeeves.py") cmd = "{runner} --input {file} --output {out} -v".format( runner=runner, file=fh.name, out=output_dir) if len(jeeves_kwargs) > 0: cmd += " %s" % dict2args(**jeeves_kwargs) logger.info("CMD: " + cmd) jeeves_log_out = "%s_out.log" % fh.name jeeves_log_err = "%s_err.log" % fh.name sh_result = sh(cmd, logout=jeeves_log_out, logerr=jeeves_log_err) result["rc"] = sh_result["rc"] if len(sh_result["status"]) > 0: result["status"] = sh_result["status"] if len(sh_result["out"]) > 0: result["status"] += sh_result["out"] if len(sh_result["err"]) > 0: result["status"] += sh_result["err"] os.remove(jeeves_log_out) os.remove(jeeves_log_err) logger.info("JEEVES_OUTPUT (JOB_ID={job})\n{sep}\n{out}\n{sep}".format( job=jds['job_id'], out=result['status'], sep='='*80)) except Exception, e: result["status"] = "EXCEPTION: " + e.__repr__() logger.error(result['status'])
def docker_is_running(container, verbose=False): tempfile = "/tmp/jeeves.%d" % os.getpid() result = sh("docker ps -a", verbose=verbose, logout=tempfile) os.remove(tempfile) if result["rc"] == ERROR_EXCEPTION: return False return container in result["out"]
def is_container_running(containerID): result = False check_result = sh("docker ps", verbose=False) assert len(check_result['err']) == 0, "got error while 'docker ps': %s" % check_result['err'] assert check_result['rc'] == 0, "got non-zero result while 'docker ps'" result = containerID[:12] in check_result['out'] return result
def get_running_commands_by_pid(): for line in sh("ps -aux").split("\n"): if "COMMAND" in line: command_index = line.index("COMMAND") continue if not line.strip(): continue pid = int(re.search(r".*?\s+(\d+)", line).group(1)) command = line[command_index:] yield pid, command
def get_connected_monitors(): xrandr_output = sh("xrandr --query").split("\n") for line in xrandr_output: if " connected " not in line: continue monitor = {"_xrandr_raw": line} monitor["labels"] = ["primary" if " primary " in line else "secondary"] (width, height) = map(int, re.findall(r"\d+x\d+", line)[0].split("x")) (x, y) = map(int, re.findall(r"\+\d+", line)) monitor.update({"width": width, "height": height, "x": x, "y": y}) yield monitor
def main(args): dir = tempfile.mkdtemp() result = util.sh("pssh -h %s -o %s uptime" % (args.hosts, dir)) if result['rc'] != 0: print "WARN: RC: %d, %s\n%s\n" % (result['rc'], result['out'], result['err']) f = glob.glob("%s/*" % dir) r = util.sh("cat " + " ".join(f)) ll = r['out'].strip().split('\n') print dir, ll avgs = map(lambda l: float(re.search("average: ([\d.]*)", l).groups()[0]), ll) shutil.rmtree(dir) num_hosts = len(avgs) avg_load = sum(avgs)/num_hosts qsize = queue_size(args.queue_name) log(args.log, (avg_load, num_hosts, args.queue_name, qsize)) if qsize == 0: return if qsize < args.min_qsize: notify_recharge(args.email, qsize) if num_hosts < args.min_hosts or avg_load < args.min_load: notify(args.email, avg_load, num_hosts)
def get_open_windows(): # TODO: add -p flag to get PID and avoid xprop invocation. # TODO: use xwininfo -tree -root to get app title. lines = sh("wmctrl -l").split("\n") hostname = sh("hostname").strip() commands_by_pid = get_running_commands_by_pid() for line in lines: if not line.strip(): continue title = re.sub(f".*? {hostname} ", "", line) window_id = re.findall(r"^0x[^\s]+", line)[0] xprop = sh(f"xprop -id {window_id}") pid = int( re.search(r"_NET_WM_PID\(CARDINAL\) = (\d+)", xprop).group(1)) yield { "_wmctrl_raw": line, "_xprop_raw": xprop, "title": title, "id": window_id, "pid": pid, # pylint: disable=unsubscriptable-object "command": commands_by_pid[pid], }
def execute_spec(window_spec, monitors_by_label): window = get_matching_window(window_spec) if not window: command_preconditions = window_spec.get("command_preconditions") check_preconditions(command_preconditions) sh(window_spec["command"], print_command=True, disown=True) window = wait_for_window(window_spec) rect = get_absolute_location(window_spec, monitors_by_label) wmctrl_location = f'0,{rect["x"]},{rect["y"]},{rect["width"]},{rect["height"]}' reposition_preconditions = window_spec.get("reposition_preconditions") check_preconditions(reposition_preconditions) sh(f'wmctrl -ia {window["id"]}', print_command=True) sh(f'wmctrl -i -r {window["id"]} -e {wmctrl_location}', print_command=True)
def docker_pull_image(image): with LockFile("/tmp/jeeves_lock_pull_%s" % re.sub("[:/]", "_", image)): result = sh("docker pull %s" % image) if result['rc'] != SUCCESS: halt("error pulling image '%s' (rc: %d, status: %s)\nERR: %s" % (image, result['rc'], result['status'], result['err']))
def run_jd_async(jd, output_basedir="output", input_basedir="input", force=False, nopull=False): global verbose JOB_ID = jd["job_id"] JOB_TAG = jd["app_container"]["name"] JOB_SUPER_ID = jd["job_super_id"] APP_CONTAINER = "JOB_%s_CNT" % JOB_SUPER_ID ENV_CONTAINER = jd["env_container"]["name"] WORK_DIR = jd["env_container"]["workdir"] CMD = jd["cmd"] ENV_VOLUMES = [] QUOTE_CMD = True if 'quote_cmd' in jd: QUOTE_CMD = jd['quote_cmd'] JOB_OUTPUT_DIR = os.path.abspath("%s/%s" % (output_basedir, JOB_ID)) JOB_INPUT_DIR = os.path.abspath("%s/%s" % (input_basedir, JOB_ID)) if 'data_volume' in jd["env_container"]: ENV_VOLUMES.append(jd["env_container"]["data_volume"]) VM_INPUT = '/input' # input folder inside VM VM_OUTPUT = '/output' # output folder inside VM ARGS = getargs(jd['args'], {"$DATA_DIR": "/data", "$OUTPUT_DIR": VM_OUTPUT, "$INPUT_DIR": VM_INPUT, "$JOB_ID": JOB_ID }) recreate_dir(JOB_OUTPUT_DIR, force=force) with open("%s/jd.json" % JOB_OUTPUT_DIR, "w") as fh: json.dump(jd, fh, indent=2, sort_keys=True) recreate_dir(JOB_INPUT_DIR, force=force) for f in jd['input_files']: assert os.path.exists(f), "input file not found: %s" % f assert os.path.isfile(f), "input file is not a regular file: %s" % f shutil.copy(f, JOB_INPUT_DIR) if not nopull: docker_pull_image(jd["app_container"]["name"]) if not docker_is_running(APP_CONTAINER): result = sh("docker run -d -v %s --name %s %s echo %s app" % (WORK_DIR, APP_CONTAINER, JOB_TAG, APP_CONTAINER), verbose=verbose) if result['rc'] != SUCCESS: halt("error running app container %s (%d, %s)" % (APP_CONTAINER, result['rc'], result['status'])) time.sleep(DELAY_DOCKER_START) ENV_VOLUMES.append("%s:%s" % (JOB_OUTPUT_DIR, VM_OUTPUT)) ENV_VOLUMES.append("%s:%s" % (JOB_INPUT_DIR, VM_INPUT)) cmd_args = "{cmd} {args}".format(cmd=CMD, args=ARGS) if QUOTE_CMD: cmd_args = "'%s'" % cmd_args docker_cmd = "docker run -d --volumes-from {app} -v {volumes} -w {workdir} {env} {cmd_args}".format( app=APP_CONTAINER, volumes=" -v ".join(ENV_VOLUMES), env=ENV_CONTAINER, workdir=WORK_DIR, cmd_args=cmd_args) time0 = datetime.datetime.now() if not nopull: docker_pull_image(ENV_CONTAINER) result = sh(docker_cmd, verbose=verbose) if result['rc'] != SUCCESS: halt("error running env container %s (%d, %s)" % (ENV_CONTAINER, result['rc'], result['status'])) containerID = result['out'].strip() while True: is_running = is_container_running(containerID) sh("docker logs %s" % containerID, logout="%s/out.log" % JOB_OUTPUT_DIR, logerr="%s/err.log" % JOB_OUTPUT_DIR, verbose=False) if not is_running: sh("docker rm %s" % containerID) break time1 = datetime.datetime.now() delay = max(DELAY_WAIT_DOCKER_MIN, min(DELAY_WAIT_DOCKER_MAX, (time1-time0).seconds * 0.1)) time.sleep(delay)
def notify_recharge(email, qsize): util.sh("/root/bin/notify.sh 'Recharge queue' current size: %d" % qsize, verbose=True)
def notify(email, avg_load, num_hosts): util.sh("/root/bin/notify.sh 'MC load average' load_avg: %f, hosts: %d" % (avg_load, num_hosts), verbose=True)