def test_is_queued(): q = JobQueue() job_spec = JobSpec( command=f'{data_path}/echo_it.bat "hello"', working_dir=pwd, log_file="test.log", priority=5, depends=[], ) job_id = q.submit([job_spec])[0] # job_id = q.submitted([job_spec]) # response = client.post("qsub", json=[job_spec.dict()]) assert job_id == JobID.parse_obj("1.0") job = q.find_job(job_id)[0] assert job.job_spec.command == job_spec.command assert job.job_spec.priority == 5 assert job.job_spec.working_dir == pwd assert job.job_spec.log_file == "test.log" assert job.job_spec.depends == [] assert len(q.queued_jobs) == 1
def qdel(job_ids, debug=False, port=config.port, ip_address=config.ip_address): """Delete jobs from the queue""" if not job_ids and sys.stdin.seekable(): # get the job ids from standard input job_id_string = "".join(c for c in sys.stdin.read() if c in digits) job_ids = job_id_string.split() if not job_ids: print("no jobs to wait on.") return config.port = port config.ip_address = ip_address # Parse the job ids input_job_ids = job_ids job_ids = [] for job_id in list(input_job_ids): if "." not in job_id: # A job group was specified response = requests.get( f"{config.url}/jobgroup?group_number={int(job_id)}") # print(response.json()) if response.status_code == 200: job_ids.extend([JobID(**item) for item in response.json()]) else: job_ids.append(JobID.parse_obj(job_id)) job_ids = [jid.dict() for jid in set(job_ids)] response = requests.post(f"{config.url}/qdel", json=job_ids) print("Jobs deleted: " + response.text)
def qpriority(priority=10, job_ids=None, port=config.port, ip_address=config.ip_address): """Change the priority of one or more jobs""" if not job_ids and sys.stdin.seekable(): # get the job ids from standard input job_id_string = "".join(c for c in sys.stdin.read() if c in digits) job_ids = job_id_string.split() if not job_ids: print("no jobs to wait on.") return config.port = port config.ip_address = ip_address # Parse the job ids input_job_ids = job_ids job_ids = [] for job_id in list(input_job_ids): if "." not in job_id: # A job group was specified response = requests.get( f"{config.url}/jobgroup?group_number={int(job_id)}") # print(response.json()) if response.status_code == 200: job_ids.extend([JobID(**item) for item in response.json()]) else: job_ids.append(JobID.parse_obj(job_id)) # job_ids = set(job_ids) job_ids = [jid.dict() for jid in set(job_ids)] # data = {"priority": priority, "job_ids": job_ids} # print(data) response = requests.post(f"{config.url}/qpriority", params={"priority": priority}, json=job_ids) print(response.text)
def qsub( command, args, priority=1, logfile=None, log=False, depends=None, debug=False, walltime=None, cores=1, port=config.port, ip_address=config.ip_address, alternate_runner=False, ): """Submits one job to the queue""" command_str = command + " " + " ".join(f'"{arg}"' for arg in args) working_dir = encode_path(os.getcwd()) if len(depends) == 1 and " " in depends[0]: depends = depends[0].split() # print(depends) if depends: depends = [JobID.parse_obj(d) for d in depends] else: depends = [] if walltime: if ":" in walltime: hrs, minutes, sec = [int(x) for x in walltime.split(":")] seconds = sec + 60 * minutes + hrs * 3600 else: walltime = float(walltime) if log and not logfile: logfile = str(Path(command).with_suffix(".lqts.log")) job_spec = JobSpec( command=command_str, working_dir=working_dir, log_file=logfile, priority=priority, depends=depends, walltime=walltime, cores=cores, alternate_runner=alternate_runner, ) config.port = port config.ip_address = ip_address response = requests.post(f"{config.url}/qsub", json=[job_spec.dict()]) if response.status_code == 200: if debug: print(response) json_data = response.json() if len(json_data) <= 20: print(" ".join(str(JobID(**item)) for item in response.json())) else: print(JobID(**json_data[0]).group) else: print(response)
def test_job_id_2(): jid1 = JobID.parse_obj("2.4") assert jid1.group == 2 assert jid1.index == 4
def qwait( job_ids=None, interval=5, port=config.port, ip_address=config.ip_address, verbose=0 ): """Blocks until the specified jobs have completed""" config.ip_address = ip_address config.port = port if not job_ids and sys.stdin.seekable(): # get the job ids from standard input job_id_string = "".join(c for c in sys.stdin.read() if c in digits) job_ids = job_id_string.split() if not job_ids: print("no jobs to wait on.") return # Parse the job ids input_job_ids = job_ids job_ids = [] for job_id in list(input_job_ids): if "." not in job_id: # A job group was specified response = requests.get(f"{config.url}/jobgroup?group_number={int(job_id)}") # print(response.json()) if response.status_code == 200: job_ids.extend([JobID(**item) for item in response.json()]) else: job_ids.append(JobID.parse_obj(job_id)) job_ids = set(job_ids) t = None done_waiting = False num_jobs_left = None while not done_waiting: options = {"completed": False} response = requests.get(f"{config.url}/qstat", json=options) # , json=message) queued_or_running_job_ids = set( Job(**ujson.loads(item)).job_id for item in response.json() ) waiting_on = job_ids.intersection(queued_or_running_job_ids) # print(waiting_on) if len(waiting_on) > 0: done_waiting = False time.sleep(interval) if t is None: if len(waiting_on) <= 20: msg = str(waiting_on) else: lsw = list(str(job) for job in sorted(waiting_on)) msg = f"{len(waiting_on)} jobs: {lsw[0]} - {lsw[-1]}" del lsw print(f"Waiting on {msg}") t = tqdm.tqdm(total=len(waiting_on)) num_jobs_left = len(waiting_on) else: num_finshed = num_jobs_left - len(waiting_on) t.update(num_finshed) num_jobs_left = len(waiting_on) else: done_waiting = True