Beispiel #1
0
def create_shortcut(config_path, icon_url=None):
    icon_url = (
        icon_url or
        "https://icons.iconarchive.com/icons/vexels/office/512/desktop-icon.png"
    )
    deskr_path = os.path.abspath(__file__)
    absolute_config_path = os.path.abspath(config_path)
    name = os.path.basename(config_path).replace(".deskr.yaml", "")
    desktop_folder = os.path.abspath(
        os.path.join(os.path.expanduser("~"), ".local/share/applications"))
    downloaded_icon_path = f"{desktop_folder}/{name}_icon.png"
    sh(f"wget -qO- {icon_url} > {downloaded_icon_path}")
    desktop_contents = f"""#!/usr/bin/env xdg-open
[Desktop Entry]
Version=1.0
Terminal=false
Type=Application
Name={name}
Exec=/usr/bin/env python3 {deskr_path} layout {absolute_config_path}
Icon={downloaded_icon_path}
"""
    desktop_path = os.path.join(desktop_folder, name + ".desktop")
    with open(desktop_path, "w") as desktop_file:
        print(f"Writing {desktop_path}")
        desktop_file.write(desktop_contents)
    sh(f"sudo chmod +x {desktop_path}", print_command=True)
Beispiel #2
0
def execute(command, *args, **kwargs):
    logger.debug("Executing: {}".format(command))
    result = sh(command, *args, **kwargs)

    if result['rc'] != SUCCESS:
        raise Exception("Error executing `{}`. result=`{}`, status=`{}`".format(command, result['rc'], result['status']))
    return result
Beispiel #3
0
def run_jd(jds, output_dir, jeeves_kwargs):
    result = {
        "rc": ERROR_EXCEPTION,
        "status": "",
        "jd": jds
    }
    time.sleep(random.random() * MAX_DELAY_BEFORE_JEEVES_START)
    logger.info("JOB_ID: %d" % jds['job_id'])
    try:
        with tempfile.NamedTemporaryFile() as fh:
            json.dump(jds, fh, indent=2, sort_keys=True)
            fh.flush()
            runner = os.path.join(my_dir(), "jeeves.py")
            cmd = "{runner} --input {file} --output {out} -v".format(
                runner=runner, file=fh.name, out=output_dir)
            if len(jeeves_kwargs) > 0: 
                cmd += " %s" % dict2args(**jeeves_kwargs)
            logger.info("CMD: " + cmd)
            jeeves_log_out = "%s_out.log" % fh.name
            jeeves_log_err = "%s_err.log" % fh.name
            sh_result = sh(cmd, logout=jeeves_log_out, logerr=jeeves_log_err)
            result["rc"] = sh_result["rc"]
            if len(sh_result["status"]) > 0:
                result["status"] = sh_result["status"]
            if len(sh_result["out"]) > 0:
                result["status"] += sh_result["out"]
            if len(sh_result["err"]) > 0:
                result["status"] += sh_result["err"]
            os.remove(jeeves_log_out)
            os.remove(jeeves_log_err)
            logger.info("JEEVES_OUTPUT (JOB_ID={job})\n{sep}\n{out}\n{sep}".format(
                job=jds['job_id'], out=result['status'], sep='='*80))
    except Exception, e:
        result["status"] = "EXCEPTION: " + e.__repr__()
        logger.error(result['status'])
Beispiel #4
0
def docker_is_running(container, verbose=False):
    tempfile = "/tmp/jeeves.%d" % os.getpid()
    result = sh("docker ps -a", verbose=verbose, logout=tempfile)
    os.remove(tempfile)
    if result["rc"] == ERROR_EXCEPTION:
        return False
    return container in result["out"]
Beispiel #5
0
def is_container_running(containerID):
    result = False
    check_result = sh("docker ps", verbose=False)
    assert len(check_result['err']) == 0, "got error while 'docker ps': %s" % check_result['err'] 
    assert check_result['rc'] == 0, "got non-zero result while 'docker ps'"
    result = containerID[:12] in check_result['out']
    return result
Beispiel #6
0
def get_running_commands_by_pid():
    for line in sh("ps -aux").split("\n"):
        if "COMMAND" in line:
            command_index = line.index("COMMAND")
            continue
        if not line.strip():
            continue
        pid = int(re.search(r".*?\s+(\d+)", line).group(1))
        command = line[command_index:]
        yield pid, command
Beispiel #7
0
def get_connected_monitors():
    xrandr_output = sh("xrandr --query").split("\n")
    for line in xrandr_output:
        if " connected " not in line:
            continue
        monitor = {"_xrandr_raw": line}
        monitor["labels"] = ["primary" if " primary " in line else "secondary"]
        (width, height) = map(int, re.findall(r"\d+x\d+", line)[0].split("x"))
        (x, y) = map(int, re.findall(r"\+\d+", line))
        monitor.update({"width": width, "height": height, "x": x, "y": y})
        yield monitor
Beispiel #8
0
def main(args):
    dir = tempfile.mkdtemp()
    result = util.sh("pssh -h %s -o %s uptime" % (args.hosts, dir))
    if result['rc'] != 0:
        print "WARN: RC: %d, %s\n%s\n" % (result['rc'], result['out'], result['err'])
    f = glob.glob("%s/*" % dir)
    r = util.sh("cat " + " ".join(f))
    ll = r['out'].strip().split('\n')
    print dir, ll
    avgs = map(lambda l: float(re.search("average: ([\d.]*)", l).groups()[0]), ll)
    shutil.rmtree(dir)
    num_hosts = len(avgs)
    avg_load = sum(avgs)/num_hosts
    qsize = queue_size(args.queue_name)

    log(args.log, (avg_load, num_hosts, args.queue_name, qsize))
    if qsize == 0:
        return
    if qsize < args.min_qsize:
        notify_recharge(args.email, qsize)
    if num_hosts < args.min_hosts or avg_load < args.min_load:
        notify(args.email, avg_load, num_hosts)
Beispiel #9
0
def get_open_windows():
    # TODO: add -p flag to get PID and avoid xprop invocation.
    # TODO: use xwininfo -tree -root to get app title.
    lines = sh("wmctrl -l").split("\n")
    hostname = sh("hostname").strip()
    commands_by_pid = get_running_commands_by_pid()

    for line in lines:
        if not line.strip():
            continue
        title = re.sub(f".*? {hostname} ", "", line)
        window_id = re.findall(r"^0x[^\s]+", line)[0]
        xprop = sh(f"xprop -id {window_id}")
        pid = int(
            re.search(r"_NET_WM_PID\(CARDINAL\) = (\d+)", xprop).group(1))
        yield {
            "_wmctrl_raw": line,
            "_xprop_raw": xprop,
            "title": title,
            "id": window_id,
            "pid": pid,
            # pylint: disable=unsubscriptable-object
            "command": commands_by_pid[pid],
        }
Beispiel #10
0
def execute_spec(window_spec, monitors_by_label):
    window = get_matching_window(window_spec)
    if not window:
        command_preconditions = window_spec.get("command_preconditions")
        check_preconditions(command_preconditions)
        sh(window_spec["command"], print_command=True, disown=True)
        window = wait_for_window(window_spec)

    rect = get_absolute_location(window_spec, monitors_by_label)
    wmctrl_location = f'0,{rect["x"]},{rect["y"]},{rect["width"]},{rect["height"]}'

    reposition_preconditions = window_spec.get("reposition_preconditions")
    check_preconditions(reposition_preconditions)

    sh(f'wmctrl -ia {window["id"]}', print_command=True)
    sh(f'wmctrl -i -r {window["id"]} -e {wmctrl_location}', print_command=True)
Beispiel #11
0
def docker_pull_image(image):
    with LockFile("/tmp/jeeves_lock_pull_%s" % re.sub("[:/]", "_", image)):
        result = sh("docker pull %s" % image)
    if result['rc'] != SUCCESS:
        halt("error pulling image '%s' (rc: %d, status: %s)\nERR: %s" %
            (image, result['rc'], result['status'], result['err']))
Beispiel #12
0
def run_jd_async(jd, output_basedir="output", input_basedir="input", force=False, nopull=False):
    global verbose
    JOB_ID = jd["job_id"]
    JOB_TAG = jd["app_container"]["name"]
    JOB_SUPER_ID = jd["job_super_id"]
    APP_CONTAINER = "JOB_%s_CNT" % JOB_SUPER_ID
    ENV_CONTAINER = jd["env_container"]["name"]
    WORK_DIR = jd["env_container"]["workdir"]
    CMD = jd["cmd"]
    ENV_VOLUMES = []
    QUOTE_CMD = True
    if 'quote_cmd' in jd:
        QUOTE_CMD = jd['quote_cmd']
    JOB_OUTPUT_DIR = os.path.abspath("%s/%s" % (output_basedir, JOB_ID))
    JOB_INPUT_DIR = os.path.abspath("%s/%s" % (input_basedir, JOB_ID))
    if 'data_volume' in jd["env_container"]:
        ENV_VOLUMES.append(jd["env_container"]["data_volume"])
    VM_INPUT = '/input'  # input folder inside VM
    VM_OUTPUT = '/output'  # output folder inside VM
    ARGS = getargs(jd['args'], 
                   {"$DATA_DIR": "/data", 
                    "$OUTPUT_DIR": VM_OUTPUT,
                    "$INPUT_DIR": VM_INPUT,
                    "$JOB_ID": JOB_ID
                    })

    recreate_dir(JOB_OUTPUT_DIR, force=force)
    with open("%s/jd.json" % JOB_OUTPUT_DIR, "w") as fh:
         json.dump(jd, fh, indent=2, sort_keys=True)
    recreate_dir(JOB_INPUT_DIR, force=force)
    for f in jd['input_files']:
        assert os.path.exists(f), "input file not found: %s" % f
        assert os.path.isfile(f), "input file is not a regular file: %s" % f
        shutil.copy(f, JOB_INPUT_DIR)

    if not nopull:
        docker_pull_image(jd["app_container"]["name"])
    if not docker_is_running(APP_CONTAINER):
        result = sh("docker run -d -v %s --name %s %s echo %s app" % 
                    (WORK_DIR, APP_CONTAINER, JOB_TAG, APP_CONTAINER),
                    verbose=verbose)
        if result['rc'] != SUCCESS:
            halt("error running app container %s (%d, %s)" % (APP_CONTAINER, result['rc'], result['status']))
        time.sleep(DELAY_DOCKER_START)
    ENV_VOLUMES.append("%s:%s" % (JOB_OUTPUT_DIR, VM_OUTPUT))
    ENV_VOLUMES.append("%s:%s" % (JOB_INPUT_DIR, VM_INPUT))

    cmd_args = "{cmd} {args}".format(cmd=CMD, args=ARGS)
    if QUOTE_CMD:
        cmd_args = "'%s'" % cmd_args
    docker_cmd = "docker run -d --volumes-from {app} -v {volumes} -w {workdir} {env} {cmd_args}".format(
        app=APP_CONTAINER, volumes=" -v ".join(ENV_VOLUMES), 
        env=ENV_CONTAINER, workdir=WORK_DIR, cmd_args=cmd_args)
    time0 = datetime.datetime.now()
    if not nopull:
        docker_pull_image(ENV_CONTAINER)
    result = sh(docker_cmd, verbose=verbose)
    if result['rc'] != SUCCESS:
        halt("error running env container %s (%d, %s)" % (ENV_CONTAINER, result['rc'], result['status']))
    containerID = result['out'].strip()
    while True:
        is_running = is_container_running(containerID)
        sh("docker logs %s" % containerID, logout="%s/out.log" % JOB_OUTPUT_DIR,
            logerr="%s/err.log" % JOB_OUTPUT_DIR, verbose=False)
        if not is_running:
            sh("docker rm %s" % containerID)
            break
        time1 = datetime.datetime.now()
        delay = max(DELAY_WAIT_DOCKER_MIN, min(DELAY_WAIT_DOCKER_MAX, (time1-time0).seconds * 0.1))
        time.sleep(delay)
Beispiel #13
0
def notify_recharge(email, qsize):
    util.sh("/root/bin/notify.sh 'Recharge queue' current size: %d" % qsize, verbose=True)
Beispiel #14
0
def notify(email, avg_load, num_hosts):
    util.sh("/root/bin/notify.sh 'MC load average' load_avg: %f, hosts: %d" % (avg_load, num_hosts), verbose=True)