def test_is_queued():
q = JobQueue()
job_spec = JobSpec(
command=f'{data_path}/echo_it.bat "hello"',
working_dir=pwd,
log_file="test.log",
priority=5,
depends=[],
)
job_id = q.submit([job_spec])[0]
# job_id = q.submitted([job_spec])
# response = client.post("qsub", json=[job_spec.dict()])
assert job_id == JobID.parse_obj("1.0")
job = q.find_job(job_id)[0]
assert job.job_spec.command == job_spec.command
assert job.job_spec.priority == 5
assert job.job_spec.working_dir == pwd
assert job.job_spec.log_file == "test.log"
assert job.job_spec.depends == []
assert len(q.queued_jobs) == 1
def qdel(job_ids, debug=False, port=config.port, ip_address=config.ip_address):
"""Delete jobs from the queue"""
if not job_ids and sys.stdin.seekable():
# get the job ids from standard input
job_id_string = "".join(c for c in sys.stdin.read() if c in digits)
job_ids = job_id_string.split()
if not job_ids:
print("no jobs to wait on.")
return
config.port = port
config.ip_address = ip_address
# Parse the job ids
input_job_ids = job_ids
job_ids = []
for job_id in list(input_job_ids):
if "." not in job_id:
# A job group was specified
response = requests.get(
f"{config.url}/jobgroup?group_number={int(job_id)}")
# print(response.json())
if response.status_code == 200:
job_ids.extend([JobID(**item) for item in response.json()])
else:
job_ids.append(JobID.parse_obj(job_id))
job_ids = [jid.dict() for jid in set(job_ids)]
response = requests.post(f"{config.url}/qdel", json=job_ids)
print("Jobs deleted: " + response.text)
def qpriority(priority=10,
job_ids=None,
port=config.port,
ip_address=config.ip_address):
"""Change the priority of one or more jobs"""
if not job_ids and sys.stdin.seekable():
# get the job ids from standard input
job_id_string = "".join(c for c in sys.stdin.read() if c in digits)
job_ids = job_id_string.split()
if not job_ids:
print("no jobs to wait on.")
return
config.port = port
config.ip_address = ip_address
# Parse the job ids
input_job_ids = job_ids
job_ids = []
for job_id in list(input_job_ids):
if "." not in job_id:
# A job group was specified
response = requests.get(
f"{config.url}/jobgroup?group_number={int(job_id)}")
# print(response.json())
if response.status_code == 200:
job_ids.extend([JobID(**item) for item in response.json()])
else:
job_ids.append(JobID.parse_obj(job_id))
# job_ids = set(job_ids)
job_ids = [jid.dict() for jid in set(job_ids)]
# data = {"priority": priority, "job_ids": job_ids}
# print(data)
response = requests.post(f"{config.url}/qpriority",
params={"priority": priority},
json=job_ids)
print(response.text)
def qsub(
command,
args,
priority=1,
logfile=None,
log=False,
depends=None,
debug=False,
walltime=None,
cores=1,
port=config.port,
ip_address=config.ip_address,
alternate_runner=False,
):
"""Submits one job to the queue"""
command_str = command + " " + " ".join(f'"{arg}"' for arg in args)
working_dir = encode_path(os.getcwd())
if len(depends) == 1 and " " in depends[0]:
depends = depends[0].split()
# print(depends)
if depends:
depends = [JobID.parse_obj(d) for d in depends]
else:
depends = []
if walltime:
if ":" in walltime:
hrs, minutes, sec = [int(x) for x in walltime.split(":")]
seconds = sec + 60 * minutes + hrs * 3600
else:
walltime = float(walltime)
if log and not logfile:
logfile = str(Path(command).with_suffix(".lqts.log"))
job_spec = JobSpec(
command=command_str,
working_dir=working_dir,
log_file=logfile,
priority=priority,
depends=depends,
walltime=walltime,
cores=cores,
alternate_runner=alternate_runner,
)
config.port = port
config.ip_address = ip_address
response = requests.post(f"{config.url}/qsub", json=[job_spec.dict()])
if response.status_code == 200:
if debug:
print(response)
json_data = response.json()
if len(json_data) <= 20:
print(" ".join(str(JobID(**item)) for item in response.json()))
else:
print(JobID(**json_data[0]).group)
else:
print(response)
def test_job_id_2():
jid1 = JobID.parse_obj("2.4")
assert jid1.group == 2
assert jid1.index == 4
def qwait(
job_ids=None, interval=5, port=config.port, ip_address=config.ip_address, verbose=0
):
"""Blocks until the specified jobs have completed"""
config.ip_address = ip_address
config.port = port
if not job_ids and sys.stdin.seekable():
# get the job ids from standard input
job_id_string = "".join(c for c in sys.stdin.read() if c in digits)
job_ids = job_id_string.split()
if not job_ids:
print("no jobs to wait on.")
return
# Parse the job ids
input_job_ids = job_ids
job_ids = []
for job_id in list(input_job_ids):
if "." not in job_id:
# A job group was specified
response = requests.get(f"{config.url}/jobgroup?group_number={int(job_id)}")
# print(response.json())
if response.status_code == 200:
job_ids.extend([JobID(**item) for item in response.json()])
else:
job_ids.append(JobID.parse_obj(job_id))
job_ids = set(job_ids)
t = None
done_waiting = False
num_jobs_left = None
while not done_waiting:
options = {"completed": False}
response = requests.get(f"{config.url}/qstat", json=options) # , json=message)
queued_or_running_job_ids = set(
Job(**ujson.loads(item)).job_id for item in response.json()
)
waiting_on = job_ids.intersection(queued_or_running_job_ids)
# print(waiting_on)
if len(waiting_on) > 0:
done_waiting = False
time.sleep(interval)
if t is None:
if len(waiting_on) <= 20:
msg = str(waiting_on)
else:
lsw = list(str(job) for job in sorted(waiting_on))
msg = f"{len(waiting_on)} jobs: {lsw[0]} - {lsw[-1]}"
del lsw
print(f"Waiting on {msg}")
t = tqdm.tqdm(total=len(waiting_on))
num_jobs_left = len(waiting_on)
else:
num_finshed = num_jobs_left - len(waiting_on)
t.update(num_finshed)
num_jobs_left = len(waiting_on)
else:
done_waiting = True