def test_parse_resource_descriptions_3():
str_res_req = ['/switch=2/nodes=10,walltime=10:0']
res_req = parse_resource_descriptions(str_res_req, default_res, nodes_res)
assert res_req == [([{u'property': u'',
u'resources': [{u'resource': u'switch', u'value': '2'},
{u'resource': 'resource_id', u'value': '10'}]}],
36000)]
def test_add_micheline_jobs_2():
default_resources = '/resource_id=1'
resource_request = parse_resource_descriptions(None, default_resources, 'resource_id')
job_vars = default_job_vars(resource_request)
job_vars['stdout'] = 'yop'
job_vars['stderr'] = 'poy'
job_vars['types'] = 'foo'
reservation_date = ''
use_job_key = False
import_job_key_inline = ''
import_job_key_file = ''
export_job_key_file = ''
initial_request = 'yop'
array_nb = 0
array_params = []
# print(job_vars)
(err, job_id_lst) = add_micheline_jobs(job_vars, reservation_date, use_job_key,
import_job_key_inline, import_job_key_file,
export_job_key_file,
initial_request, array_nb, array_params)
print("job id:", job_id_lst)
assert len(job_id_lst) == 1
def test_add_micheline_simple_array_job():
default_resources = 'network_address=2/resource_id=1+/resource_id=2'
resource_request = parse_resource_descriptions(None, default_resources, 'resource_id')
job_vars = default_job_vars(resource_request)
job_vars['types'] = 'foo'
reservation_date = ''
use_job_key = False
import_job_key_inline = ''
import_job_key_file = ''
export_job_key_file = ''
initial_request = 'yop'
array_nb = 5
array_params = [str(i) for i in range(array_nb)]
# print(job_vars)
(err, job_id_lst) = add_micheline_jobs(job_vars, reservation_date, use_job_key,
import_job_key_inline, import_job_key_file,
export_job_key_file,
initial_request, array_nb, array_params)
r = db['JobResourceGroup'].query.all()
for item in r:
print(item.to_dict())
r = db['JobResourceDescription'].query.all()
for item in r:
print(item.to_dict())
print("job id:", job_id_lst)
assert len(job_id_lst) == 5
def test_parse_resource_descriptions_4():
str_res_req = ["{gpu='YES'}/nodes=ALL+{gpu='NO'}/core=20"]
res_req = parse_resource_descriptions(str_res_req, default_res, nodes_res)
assert res_req == [([{u'property': u"gpu='YES'",
u'resources': [{u'resource': 'resource_id', u'value': -1}]},
{u'property': u"gpu='NO'",
u'resources': [{u'resource': u'core', u'value': '20'}]}],
None)]
def test_parse_resource_descriptions_1():
str_res_req = ['/resource_id=1']
res_req = parse_resource_descriptions(str_res_req, default_res, nodes_res)
assert res_req == [([{u'property': u'',
u'resources': [{u'resource': u'resource_id', u'value': '1'}]}],
None)]
def cli(
command,
interactive,
queue,
resource,
reservation,
connect,
type,
checkpoint,
property,
resubmit,
scanscript,
project,
signal,
directory,
name,
after,
notify,
array,
array_param_file,
use_job_key,
import_job_key_from_file,
import_job_key_inline,
export_job_key_to_file,
stdout,
stderr,
hold,
version,
):
"""Submit a job to OAR batch scheduler."""
config.setdefault_config(DEFAULT_CONFIG)
# import pdb; pdb.set_trace()
# print(resource)
# When the walltime of a job is not defined
default_job_walltime = str(config["DEFAULT_JOB_WALLTIME"])
log_warning = "" # TODO
log_error = ""
log_info = ""
log_std = ""
remote_host = config["SERVER_HOSTNAME"]
remote_port = int(config["SERVER_PORT"])
if "OARSUB_DEFAULT_RESOURCES" in config:
default_resources = config["OARSUB_DEFAULT_RESOURCES"]
else:
default_resources = "/resource_id=1"
if "OARSUB_NODES_RESOURCES" in config:
nodes_resources = config["OARSUB_NODES_RESOURCES"]
else:
nodes_resources = "resource_id"
# TODO Deploy_hostname / Cosystem_hostname
# $Deploy_hostname = get_conf("DEPLOY_HOSTNAME");
# if (!defined($Deploy_hostname)){
# $Deploy_hostname = $remote_host;
# }
# $Cosystem_hostname = get_conf("COSYSTEM_HOSTNAME");
# if (!defined($Cosystem_hostname)){
# $Cosystem_hostname = $remote_host;
# }
cpuset_field = config["JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD"]
cpuset_path = config["CPUSET_PATH"]
if "OAR_RUNTIME_DIRECTORY" in config:
pass
# if (is_conf("OAR_RUNTIME_DIRECTORY")){
# OAR::Sub::set_default_oarexec_directory(get_conf("OAR_RUNTIME_DIRECTORY"));
# }
# my $default_oar_dir = OAR::Sub::get_default_oarexec_directory();
# if (!(((-d $default_oar_dir) and (-O $default_oar_dir)) or (mkdir($default_oar_dir)))){
# die("# Error: failed to create the OAR directory $default_oar_dir, or bad permissions.\n");
# }
binpath = ""
if "OARDIR" in os.environ:
binpath = os.environ["OARDIR"] + "/"
else:
print_error("OARDIR environment variable is not defined.")
sub_exit(1)
openssh_cmd = config["OPENSSH_CMD"]
ssh_timeout = int(config["OAR_SSH_CONNECTION_TIMEOUT"])
# if (is_conf("OAR_SSH_CONNECTION_TIMEOUT")){
# OAR::Sub::set_ssh_timeout(get_conf("OAR_SSH_CONNECTION_TIMEOUT"));
# }
# OAR version
# TODO: OAR is now a set of composition...
#
types = type
properties = lstrip_none(property)
if not directory:
launching_directory = ""
else:
launching_directory = lstrip_none(directory)
initial_request = " ".join(sys.argv[1:])
queue_name = lstrip_none(queue)
reservation_date = lstrip_none(reservation)
if reservation_date:
m = re.search(r"^\s*(\d{4}\-\d{1,2}\-\d{1,2})\s+(\d{1,2}:\d{1,2}:\d{1,2})\s*$", reservation)
if m:
reservation_date = sql_to_local(m.group(1) + " " + m.group(2))
else:
print_error(
'syntax error for the advance reservation start date \
specification. Expected format is:"YYYY-MM-DD hh:mm:ss"'
)
sub_exit(7)
if array:
array_nb = array
else:
array_nb = 1
# Check the default name of the key if we have to generate it
if ("OARSUB_FORCE_JOB_KEY" in config) and config["OARSUB_FORCE_JOB_KEY"] == "yes":
use_job_key = True
else:
use_job_key = False
# TODO ssh_private_key, ssh_public_key,
# ssh_private_key = ''
# ssh_public_key = ''
# TODO import_job_key_file, export_job_key_file
import_job_key_file = ""
export_job_key_file = ""
if resubmit:
print("# Resubmitting job ", resubmit, "...")
ret = resubmit_job(resubmit)
if ret > 0:
job_id = ret
print(" done.\n")
print("OAR_JOB_ID=" + str(job_id))
if signal_almighty(remote_host, remote_port, "Qsub") > 0:
print_error(
"cannot connect to executor "
+ str(remote_host)
+ ":"
+ str(remote_port)
+ ". OAR server might be down."
)
sub_exit(3)
else:
sub_exit(0)
else:
print(" error.")
if ret == -1:
print_error("interactive jobs and advance reservations cannot be resubmitted.")
elif ret == -2:
print_error("only jobs in the Error or Terminated state can be resubmitted.")
elif ret == -3:
print_error("resubmitted job user mismatch.")
elif ret == -4:
print_error("another active job is using the same job key.")
else:
print_error("unknown error.")
sub_exit(4)
if not command and not interactive and not reservation and not connect:
usage()
sub_exit(5)
if interactive and reservation:
print_error("an advance reservation cannot be interactive.")
usage()
sub_exit(7)
if interactive and any(re.match(r"^desktop_computing$", t) for t in type):
print_error(" a desktop computing job cannot be interactive")
usage()
sub_exit(17)
if any(re.match(r"^noop$", t) for t in type):
if interactive:
print_error("a NOOP job cannot be interactive.")
sub_exit(17)
elif connect:
print_error("a NOOP job does not have a shell to connect to.")
sub_exit(17)
# notify : check insecure character
if notify and re.match(r"^.*exec\s*:.+$"):
m = re.search(r".*exec\s*:([a-zA-Z0-9_.\/ -]+)$", notify)
if not m:
print_error(
"insecure characters found in the notification method \
(the allowed regexp is: [a-zA-Z0-9_.\/ -]+)."
)
sub_exit(16)
# TODO Connect to a reservation
# Connect to a reservation
# if (defined($connect_job)){
# Do not kill the job if the user close the window
# $SIG{HUP} = 'DEFAULT';
# OAR::Sub::close_db_connection(); exit(connect_job($connect_job,0,$Openssh_cmd));
# }
if not project:
project = DEFAULT_VALUE["project"]
if not signal:
signal = DEFAULT_VALUE["signal"]
if not directory:
directory = DEFAULT_VALUE["directory"]
resource_request = parse_resource_descriptions(resource, default_resources, nodes_resources)
job_vars = {
"job_type": None,
"resource_request": resource_request,
"command": command,
"info_type": None,
"queue_name": queue_name,
"properties": properties,
"checkpoint": checkpoint,
"signal": signal,
"notify": notify,
"name": name,
"types": types,
"launching_directory": launching_directory,
"dependencies": after,
"stdout": stdout,
"stderr": stderr,
"hold": hold,
"project": project,
"initial_request": initial_request,
"user": os.environ["OARDO_USER"],
"array_id": 0,
"start_time": "0",
"reservation_field": None,
}
if not interactive and command:
cmd_executor = "Qsub"
if scanscript:
# TODO scanscript
pass
array_params = []
if array_param_file:
pass
# TODO
# $array_params_ref = OAR::Sub::read_array_param_file($array_param_file);
# $array_nb = scalar @{$array_params_ref};
if array_nb == 0:
print_error("an array of job must have a number of sub-jobs greater than 0.")
usage()
sub_exit(6)
job_vars["info_type"] = "$Host:$server_port" # TODO "$Host:$server_port"
job_vars["job_type"] = "PASSIVE"
(err, job_id_lst) = add_micheline_jobs(
job_vars,
reservation_date,
use_job_key,
import_job_key_inline,
import_job_key_file,
export_job_key_file,
initial_request,
array_nb,
array_params,
)
else:
# TODO interactive
if command:
print_warning("asking for an interactive job (-I), so ignoring arguments: " + command + " .")
cmd_executor = "Qsub -I"
if array_param_file:
print_error("a array job with parameters given in a file cannot be interactive.")
usage()
sub_exit(9)
if array_nb != 1:
print_error("an array job cannot be interactive.")
usage()
sub_exit(8)
if reservation:
# Test if this job is a reservation and the syntax is right
# TODO Pass
pass
socket_server = init_tcp_server()
(server, server_port) = socket_server.getsockname()
job_vars["info_type"] = server + ":" + str(server_port)
job_vars["job_type"] = "INTERACTIVE"
(err, job_id_lst) = add_micheline_jobs(
job_vars,
reservation_date,
use_job_key,
import_job_key_inline,
import_job_key_file,
export_job_key_file,
initial_request,
array_nb,
array_params,
)
# pdb.set_trace()
if err != 0:
print_error("command failed, please verify your syntax.")
sub_exit(err, "")
oar_array_id = 0
# Print job_id list
if len(job_id_lst) == 1:
print("OAR_JOB_ID=", job_id_lst[0])
else:
job = db["Job"].query.filter(Job.id == job_id_lst[0]).one()
oar_array_id = job.array_id
for job_id in job_id_lst:
print("OAR_JOB_ID=", job_id)
result = (job_id_lst, oar_array_id)
# Notify Almigthy
tools.create_almighty_socket()
tools.notify_almighty(cmd_executor)
if reservation:
# Reservation mode
print_info("advance reservation request: waiting for approval from the scheduler...")
(conn, address) = socket_server.accept()
answer = conn.recv(1024)
if answer[:-1] == "GOOD RESERVATION":
print_info("advance reservation is GRANTED.")
else:
print_info("advance reservation is REJECTED ", answer[:-1])
sub_exit(10)
elif interactive:
# Interactive mode
print_info("interactive mode: waiting...")
prev_str = ""
while True:
(conn, address) = socket_server.accept()
answer = conn.recv(1024)
answer = answer[:-1]
m = re.search(r"\](.*)$", answer)
if m and m.group(1) != prev_str:
print_info(answer)
prev_str = m.group(1)
elif answer != "GOOD JOB":
print_info(answer)
if (
(answer == "GOOD JOB")
or (answer == "BAD JOB")
or (answer == "JOB KILLE")
or re.match(r"^ERROR", answer)
):
break
if answer == "GOOD JOB":
# TODO exit(connect_job($Job_id_list_ref->[0],1,$Openssh_cmd));
pass
else:
sub_exit(11)
sub_exit(0, result)