def fetch_job_list(cls, hpc, token, proxies={}): """ Retrieve job list from Unicore systems """ job_list_dict = collections.OrderedDict() auth = unicore_client.get_oidc_auth(token="Bearer " + token) base_url = unicore_client.get_sites()[hpc]['url'] if hpc == "DAINT-CSCS": listofjobs = unicore_client.get_properties(base_url + '/jobs', auth, \ proxies=proxies) jobs = listofjobs['jobs'] for i in jobs: job_title = i.split('/')[-1] job_list_dict[job_title] = collections.OrderedDict() job_list_dict[job_title]['url'] = i elif hpc == "SA-CSCS": listofjobs = unicore_client.get_properties(base_url + '/jobs/pizdaint/bsp_pizdaint_01/', auth, \ proxies=proxies) jobs = listofjobs for i in jobs: job_title = i["job_id"] job_list_dict[job_title] = collections.OrderedDict() job_list_dict[job_title][ 'url'] = base_url + '/jobs/pizdaint/bsp_pizdaint_01/' + job_title return job_list_dict
def fetch_job_results(cls, hpc="", job_url="", token="", dest_dir="", proxies={}): """ """ # create destination dir if not existing if not os.path.exists(dest_dir): os.mkdir(dest_dir) auth = unicore_client.get_oidc_auth(token=token) r = unicore_client.get_properties(job_url, auth, proxies=proxies) # job retrieving from SA-CSCS if hpc == "SA-CSCS": r = requests.get(url=job_url, headers=auth) if r.status_code == 200: job = r.json() job_id = job_url.split("/")[-1] job_files_url = cls.SA_CSCS_FILES_URL + job_id + "/" r = requests.get(url=job_files_url, headers=auth) if r.status_code == 200: files_list = r.json() for f in files_list: if f == '/stderr' or f == '/stdout' or f == '/output.zip': filename = f[1:] fname, extension = os.path.splitext(filename) crr_file_url = job_files_url + filename + '/' r = requests.get(url=crr_file_url, headers=auth) if r.status_code == 200: with open(os.path.join(dest_dir, filename), 'w') as local_file: local_file.write(r.content) elif hpc == "DAINT-CSCS": if (r['status'] == 'SUCCESSFUL') or (r['status'] == 'FAILED'): wd = unicore_client.get_working_directory(job_url, auth, proxies=proxies) output_files = unicore_client.list_files(wd, auth, proxies=proxies) for file_path in output_files: _, f = os.path.split(file_path) if (f == 'stderr') or (f == "stdout") or (f == "output.zip"): content = unicore_client.get_file_content(wd + "/files" + \ file_path, auth, MAX_SIZE=cls.MAX_SIZE, proxies=proxies) with open(os.path.join(dest_dir, f), "w") as local_file: local_file.write(content) local_file.close() return ""
def checkLogin(cls, username="", token="", hpc="", proxies={}): auth = unicore_client.get_oidc_auth(token=token) base_url = unicore_client.get_sites()[hpc]['url'] role = unicore_client.get_properties( base_url, auth, proxies)['client']['role']['selected'] resp = {"response" : "KO", "message" : "This account is not " + \ "allowed to submit job on " + hpc + " booster partition"} # if role == "user": info = unicore_client.get_properties( base_url, auth, proxies=proxies)['client']['xlogin']['UID'] UIDs = unicore_client.get_properties( base_url, auth, proxies=proxies)['client']['xlogin']['availableUIDs'] if username in UIDs and username[0:3] == "vsk": resp = { "response": "OK", "message": "Successful authentication" } return resp
def fetch_job_details(cls, hpc="", job_url="", job_id="", token="", proxies={}): """ """ # base_url = unicore_client.get_sites()[hpc]['url'] auth = unicore_client.get_oidc_auth(token=token) # mapping for DAINT-CSCS and SA-CSCS fields mapfield = { "job_date_submitted": {"DAINT-CSCS":'submissionTime',\ "SA-CSCS":"init_date"}, "job_stage": {"DAINT-CSCS":'status',\ "SA-CSCS":"stage"}, "job_res_url": {"DAINT-CSCS":job_url,\ "SA-CSCS":"https://bspsa.cineca.it/files/pizdaint/bsp_pizdaint_01/" + job_url}, "job_name": {"DAINT-CSCS":'name',\ "SA-CSCS":"title"}, "status_message": {"DAINT-CSCS":'statusMessage',\ "SA-CSCS":"stage"}, } job_details = unicore_client.get_properties(job_url, auth, proxies=proxies) job_info_dict = collections.OrderedDict() job_info_dict["job_date_submitted"] = job_details[ mapfield["job_date_submitted"][hpc]] job_info_dict["job_stage"] = job_details[mapfield["job_stage"][hpc]] job_info_dict["job_id"] = job_id job_info_dict["job_res_url"] = job_url job_info_dict["job_name"] = job_details[mapfield["job_name"][hpc]] job_info_dict["status_message"] = job_details[ mapfield["status_message"][hpc]] return job_info_dict