def fetch_job_list(cls, hpc, token, proxies={}):
"""
Retrieve job list from Unicore systems
"""
job_list_dict = collections.OrderedDict()
auth = unicore_client.get_oidc_auth(token="Bearer " + token)
base_url = unicore_client.get_sites()[hpc]['url']
if hpc == "DAINT-CSCS":
listofjobs = unicore_client.get_properties(base_url + '/jobs', auth, \
proxies=proxies)
jobs = listofjobs['jobs']
for i in jobs:
job_title = i.split('/')[-1]
job_list_dict[job_title] = collections.OrderedDict()
job_list_dict[job_title]['url'] = i
elif hpc == "SA-CSCS":
listofjobs = unicore_client.get_properties(base_url + '/jobs/pizdaint/bsp_pizdaint_01/', auth, \
proxies=proxies)
jobs = listofjobs
for i in jobs:
job_title = i["job_id"]
job_list_dict[job_title] = collections.OrderedDict()
job_list_dict[job_title][
'url'] = base_url + '/jobs/pizdaint/bsp_pizdaint_01/' + job_title
return job_list_dict
def fetch_job_results(cls,
hpc="",
job_url="",
token="",
dest_dir="",
proxies={}):
"""
"""
# create destination dir if not existing
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
auth = unicore_client.get_oidc_auth(token=token)
r = unicore_client.get_properties(job_url, auth, proxies=proxies)
# job retrieving from SA-CSCS
if hpc == "SA-CSCS":
r = requests.get(url=job_url, headers=auth)
if r.status_code == 200:
job = r.json()
job_id = job_url.split("/")[-1]
job_files_url = cls.SA_CSCS_FILES_URL + job_id + "/"
r = requests.get(url=job_files_url, headers=auth)
if r.status_code == 200:
files_list = r.json()
for f in files_list:
if f == '/stderr' or f == '/stdout' or f == '/output.zip':
filename = f[1:]
fname, extension = os.path.splitext(filename)
crr_file_url = job_files_url + filename + '/'
r = requests.get(url=crr_file_url, headers=auth)
if r.status_code == 200:
with open(os.path.join(dest_dir, filename),
'w') as local_file:
local_file.write(r.content)
elif hpc == "DAINT-CSCS":
if (r['status'] == 'SUCCESSFUL') or (r['status'] == 'FAILED'):
wd = unicore_client.get_working_directory(job_url,
auth,
proxies=proxies)
output_files = unicore_client.list_files(wd,
auth,
proxies=proxies)
for file_path in output_files:
_, f = os.path.split(file_path)
if (f == 'stderr') or (f == "stdout") or (f
== "output.zip"):
content = unicore_client.get_file_content(wd + "/files" + \
file_path, auth, MAX_SIZE=cls.MAX_SIZE, proxies=proxies)
with open(os.path.join(dest_dir, f),
"w") as local_file:
local_file.write(content)
local_file.close()
return ""
def checkLogin(cls, username="", token="", hpc="", proxies={}):
auth = unicore_client.get_oidc_auth(token=token)
base_url = unicore_client.get_sites()[hpc]['url']
role = unicore_client.get_properties(
base_url, auth, proxies)['client']['role']['selected']
resp = {"response" : "KO", "message" : "This account is not " + \
"allowed to submit job on " + hpc + " booster partition"}
#
if role == "user":
info = unicore_client.get_properties(
base_url, auth, proxies=proxies)['client']['xlogin']['UID']
UIDs = unicore_client.get_properties(
base_url, auth,
proxies=proxies)['client']['xlogin']['availableUIDs']
if username in UIDs and username[0:3] == "vsk":
resp = {
"response": "OK",
"message": "Successful authentication"
}
return resp
def fetch_job_details(cls,
hpc="",
job_url="",
job_id="",
token="",
proxies={}):
"""
"""
# base_url = unicore_client.get_sites()[hpc]['url']
auth = unicore_client.get_oidc_auth(token=token)
# mapping for DAINT-CSCS and SA-CSCS fields
mapfield = {
"job_date_submitted":
{"DAINT-CSCS":'submissionTime',\
"SA-CSCS":"init_date"},
"job_stage":
{"DAINT-CSCS":'status',\
"SA-CSCS":"stage"},
"job_res_url":
{"DAINT-CSCS":job_url,\
"SA-CSCS":"https://bspsa.cineca.it/files/pizdaint/bsp_pizdaint_01/" + job_url},
"job_name":
{"DAINT-CSCS":'name',\
"SA-CSCS":"title"},
"status_message":
{"DAINT-CSCS":'statusMessage',\
"SA-CSCS":"stage"},
}
job_details = unicore_client.get_properties(job_url,
auth,
proxies=proxies)
job_info_dict = collections.OrderedDict()
job_info_dict["job_date_submitted"] = job_details[
mapfield["job_date_submitted"][hpc]]
job_info_dict["job_stage"] = job_details[mapfield["job_stage"][hpc]]
job_info_dict["job_id"] = job_id
job_info_dict["job_res_url"] = job_url
job_info_dict["job_name"] = job_details[mapfield["job_name"][hpc]]
job_info_dict["status_message"] = job_details[
mapfield["status_message"][hpc]]
return job_info_dict