def fetch_job_list(cls, hpc, token, proxies={}): """ Retrieve job list from Unicore systems """ job_list_dict = collections.OrderedDict() auth = unicore_client.get_oidc_auth(token="Bearer " + token) base_url = unicore_client.get_sites()[hpc]['url'] if hpc == "DAINT-CSCS": listofjobs = unicore_client.get_properties(base_url + '/jobs', auth, \ proxies=proxies) jobs = listofjobs['jobs'] for i in jobs: job_title = i.split('/')[-1] job_list_dict[job_title] = collections.OrderedDict() job_list_dict[job_title]['url'] = i elif hpc == "SA-CSCS": listofjobs = unicore_client.get_properties(base_url + '/jobs/pizdaint/bsp_pizdaint_01/', auth, \ proxies=proxies) jobs = listofjobs for i in jobs: job_title = i["job_id"] job_list_dict[job_title] = collections.OrderedDict() job_list_dict[job_title][ 'url'] = base_url + '/jobs/pizdaint/bsp_pizdaint_01/' + job_title return job_list_dict
def fetch_job_results(cls, hpc="", job_url="", token="", dest_dir="", proxies={}): """ """ # create destination dir if not existing if not os.path.exists(dest_dir): os.mkdir(dest_dir) auth = unicore_client.get_oidc_auth(token=token) r = unicore_client.get_properties(job_url, auth, proxies=proxies) # job retrieving from SA-CSCS if hpc == "SA-CSCS": r = requests.get(url=job_url, headers=auth) if r.status_code == 200: job = r.json() job_id = job_url.split("/")[-1] job_files_url = cls.SA_CSCS_FILES_URL + job_id + "/" r = requests.get(url=job_files_url, headers=auth) if r.status_code == 200: files_list = r.json() for f in files_list: if f == '/stderr' or f == '/stdout' or f == '/output.zip': filename = f[1:] fname, extension = os.path.splitext(filename) crr_file_url = job_files_url + filename + '/' r = requests.get(url=crr_file_url, headers=auth) if r.status_code == 200: with open(os.path.join(dest_dir, filename), 'w') as local_file: local_file.write(r.content) elif hpc == "DAINT-CSCS": if (r['status'] == 'SUCCESSFUL') or (r['status'] == 'FAILED'): wd = unicore_client.get_working_directory(job_url, auth, proxies=proxies) output_files = unicore_client.list_files(wd, auth, proxies=proxies) for file_path in output_files: _, f = os.path.split(file_path) if (f == 'stderr') or (f == "stdout") or (f == "output.zip"): content = unicore_client.get_file_content(wd + "/files" + \ file_path, auth, MAX_SIZE=cls.MAX_SIZE, proxies=proxies) with open(os.path.join(dest_dir, f), "w") as local_file: local_file.write(content) local_file.close() return ""
def fetch_job_details(cls, hpc="", job_url="", job_id="", token="", proxies={}): """ """ # base_url = unicore_client.get_sites()[hpc]['url'] auth = unicore_client.get_oidc_auth(token=token) # mapping for DAINT-CSCS and SA-CSCS fields mapfield = { "job_date_submitted": {"DAINT-CSCS":'submissionTime',\ "SA-CSCS":"init_date"}, "job_stage": {"DAINT-CSCS":'status',\ "SA-CSCS":"stage"}, "job_res_url": {"DAINT-CSCS":job_url,\ "SA-CSCS":"https://bspsa.cineca.it/files/pizdaint/bsp_pizdaint_01/" + job_url}, "job_name": {"DAINT-CSCS":'name',\ "SA-CSCS":"title"}, "status_message": {"DAINT-CSCS":'statusMessage',\ "SA-CSCS":"stage"}, } job_details = unicore_client.get_properties(job_url, auth, proxies=proxies) job_info_dict = collections.OrderedDict() job_info_dict["job_date_submitted"] = job_details[ mapfield["job_date_submitted"][hpc]] job_info_dict["job_stage"] = job_details[mapfield["job_stage"][hpc]] job_info_dict["job_id"] = job_id job_info_dict["job_res_url"] = job_url job_info_dict["job_name"] = job_details[mapfield["job_name"][hpc]] job_info_dict["status_message"] = job_details[ mapfield["status_message"][hpc]] return job_info_dict
def checkLogin(cls, username="", token="", hpc="", proxies={}): auth = unicore_client.get_oidc_auth(token=token) base_url = unicore_client.get_sites()[hpc]['url'] role = unicore_client.get_properties( base_url, auth, proxies)['client']['role']['selected'] resp = {"response" : "KO", "message" : "This account is not " + \ "allowed to submit job on " + hpc + " booster partition"} # if role == "user": info = unicore_client.get_properties( base_url, auth, proxies=proxies)['client']['xlogin']['UID'] UIDs = unicore_client.get_properties( base_url, auth, proxies=proxies)['client']['xlogin']['availableUIDs'] if username in UIDs and username[0:3] == "vsk": resp = { "response": "OK", "message": "Successful authentication" } return resp
def run_unicore_opt(cls, hpc="", filename="", joblaunchname = "", \ token = None, core_num = 4, jobname="UNICORE_Job", \ node_num = 2, runtime = 4, username="", foldname = "", proxies={}): basename = os.path.basename(filename) # read file to be moved to remote with open(filename, 'r') as f: content = f.read() f.close() modpy = content mod = {'To': basename, 'Data': modpy} # create inputs array inputs = [mod] # define exec string depending on hpc system if hpc == "DAINT-CSCS": exec_str = "; chmod +rx *.sbatch; ./ipyparallel.sbatch" hpc_sub = "DAINT-CSCS" # create job to be submitted job = {} job = { "Executable": "unzip " + basename + "; cd " + foldname + exec_str, "Name": jobname, "Resources": { "Nodes": str(node_num), \ "CPUsPerNode": str(core_num), \ "Runtime": runtime, \ "NodeConstraints": "mc", }, } auth = unicore_client.get_oidc_auth(token=token) # auth['X-UNICORE-User-Preferences'] = 'uid:'+ username base_url = unicore_client.get_sites()[hpc_sub]['url'] elif hpc == "SA-CSCS": exec_str = "; chmod +rx *.sbatch; ./ipyparallel.sbatch" command = "unzip " + basename + "; cd " + foldname + exec_str payload = { 'command': command, 'node_number': str(node_num), 'core_number': str(core_num), 'runtime': str(runtime), } job_file = {'file': open(filename, 'rb')} auth = unicore_client.get_oidc_auth(token=token) auth.update( {'Content-Disposition': 'attachment;filename=' + basename}) auth.update({'payload': json.dumps(payload)}) elif hpc == "jureca": exec_str = "; chmod +rx *.sh; chmod +rx *.sbatch; sh " + joblaunchname + ";", hpc_sub = "jureca" try: if hpc == "SA-CSCS": SUBMIT_URL = 'https://bspsa.cineca.it/jobs/pizdaint/' r = requests.post(url=SUBMIT_URL, headers=auth, files=job_file) job = r.json() jobname = job['job_id'] job_url = 'https://bspsa.cineca.it/jobs/pizdaint/bsp_pizdaint_01/' + jobname + '/' else: job_url = unicore_client.submit(base_url + '/jobs', job, auth, \ inputs, proxies=proxies) jobname = job_url.split('/')[-1] resp = {'response':'OK', 'joburl':job_url, 'jobname': jobname, \ 'job_id': jobname, 'message':'Job submitted with success', 'status_code':200} except Exception as e: print(e) resp = {'response': 'KO', 'message': 'Operation not completed'} return resp