def execute(request, machine_name, command): try: logger.debug("Running command: %s" % command) #print( command + 's') res = run_command(command) (output, error, retcode) = run_command(command) response = {'output': output, 'error': error, 'retcode': retcode} #print( response) return response except Exception as e: logger.error("Could not run command: %s" % str(e)) return json_response(error="Could not run command: %s" % str(e), status="ERROR", status_code=500)
def ping(machine_name, hostname): # Do a single ping with timeout 2 seconds (output, err, retcode) = run_command('ping -c1 -t2 %s' % hostname) if retcode == 0: return {'system': machine_name, 'status': 'up'} else: return {'system': machine_name, 'status': 'down'}
def download_path(request, machine_name, path): """Returns a StreamingHttpResponse with the file Keyword arguments: machine_name -- name of the machine path -- path to file """ src = gridutil.get_grid_path(machine_name, path) env = gridutil.get_cred_env(request.user) tmp_file = tempfile.NamedTemporaryFile(prefix="newt_") dest = "file://" + tmp_file.name logger.debug("File download requested: %s (%s)" % (path, src)) (output, error, retcode) = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-url-copy %s %s" % (src, dest), env=env) if retcode != 0: logger.warning("Unable to download file: %s" % src) return json_response(content=output, status="ERROR", status_code=500, error=error) # filename = path.rsplit("/")[-1] # f = open(dest+"filename", "r") mimetype = mimetypes.guess_type(tmp_file.name) if mimetype is None: mimetype = "application/octet-stream" return StreamingHttpResponse(tmp_file, content_type=mimetype)
def submit_job_task_unsafty(self, taskenv, HPCJobid, jobfilepath): pass try: job = HPCJob.objects.get(id=HPCJobid) except ImportError: return sys.path if job.jobid: return worker_json_response(status="ERROR", status_code=500, error="The Job have submit once : %s" % job.jobid) if job.state == "tempfile": # mv jobfile to user dir src = os.path.basename(job.jobfile) temphost = taskenv["host"] if jobfilepath: dest = jobfilepath else: dest = os.path.join( getpwnam(job.user.username).pw_dir, 'newt', str(job.id), os.path.basename(job.jobfile)) dest_dir = os.path.dirname(dest) if not os.path.isdir(dest_dir): os.makedirs(dest_dir) put_file_task(taskenv, temphost, src, dest) pass #dest = os.path.join( getpwnam( job.user.username ).pw_dir , 'newt' , str(job.id) , os.path.basename( job.jobfile ) ) if dest.startswith('~'): dest = os.path.join(getpwnam(job.user.username).pw_dir, dest[2:]) if not os.path.isfile(dest): return worker_json_response(status="ERROR", status_code=500, error="Cannot find tmpfile : %s" % dest) else: job.jobfile = dest job.state = "unsubmit" job.save() # can not save as readonly qsub_conf = slurmutil.GRID_RESOURCE_TABLE.get( taskenv["machine"]).get('qsub') if qsub_conf.get('parser_cls', False): cmd_sub_arg = qsub_conf["parser_cls"].gen_args(job) else: cmd_sub_arg = "" #cmd_sub_arg = qsub_conf.parser_cls.gen_args( job ) if qsub_conf.get( 'parser_cls' , False ) else "" qsub = qsub_conf['bin'] #cmd_str = ''' bash -c -l "%s %s %s" ''' % (qsub, job.jobfile , job.jobfile_args ) cmd_str = "%s %s %s %s" % (qsub, cmd_sub_arg, job.jobfile, job.jobfile_args) os.environ['PWD'] = os.path.dirname(job.jobfile) os.chdir(os.path.dirname(job.jobfile)) (output, error, retcode) = run_command(cmd_str, bash=True) if retcode != 0: return worker_json_response(status="ERROR", status_code=500, error="qsub failed with error: %s" % error) job.jobid = output.strip().split(' ')[-1] job.state = "submited" job.save() # can not save as readonly return worker_json_response({"jobid": job.jobid})
def execute(request, machine_name, command): """Returns a the result of running command on machine_name Keyword arguments: machine_name -- name of the machine command -- command to run """ machine = gridutil.GRID_RESOURCE_TABLE.get(machine_name, None) if not machine: return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name) # Get the grid credentials for the user env = gridutil.get_cred_env(request.user) try: # Run the command using globus-job-run (output, error, retcode) = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-job-run %s %s" % (machine['hostname'], command), env=env) response = { "output": output, "error": error, "retcode": retcode } return response except Exception as e: logger.error("Could not run command: %s" % str(e)) return json_response(error="Could not run command: %s" % str(e), status="ERROR", status_code=500)
def view_queue(request, machine_name): """Returns the current state of the queue in a list Keyword arguments: request -- Django HttpRequest machine_name -- name of the machine """ machine = gridutil.GRID_RESOURCE_TABLE.get(machine_name, None) if not machine: return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name) env = gridutil.get_cred_env(request.user) (output, error, retcode) = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-job-run %s /project/projectdirs/osp/newt_tools/qs_moab.sh" % (machine['hostname']), env=env) patt = re.compile(r'(?P<jobid>[^\s]+)\s+(?P<status>[^\s]+)\s+(?P<user>[^\s]+)\s+(?P<job_name>[^\s]+)\s+(?P<nodes>\d+)\s+(?P<walltime>[^\s]+)\s+(?P<time_use>[^\s]+)\s+(?P<time_submit>\w{3}\s\d{1,2}\s[\d\:]+)\s+(?P<rank>[^\s]+)\s+(?P<queue>[^\s]+)\s+(?P<q_state>[^\s]+)\s+(?P<processors>[^\s]+)\s*(?P<details>.*)$') if retcode != 0: return json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error) # filter out stuff that doesn't match pattern output = output.splitlines() output = [x.strip() for x in output] output = filter(lambda line: patt.match(line), output) # Convert output into dict from group names output = map(lambda x: patt.match(x).groupdict(), output) return output
def execute(request, machine_name, command): """Returns a the result of running command on machine_name Keyword arguments: machine_name -- name of the machine command -- command to run """ machine = gridutil.GRID_RESOURCE_TABLE.get(machine_name, None) if not machine: return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name) # Get the grid credentials for the user env = gridutil.get_cred_env(request.user) try: # Run the command using globus-job-run (output, error, retcode) = run_command( gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-job-run %s %s" % (machine['hostname'], command), env=env) response = {"output": output, "error": error, "retcode": retcode} return response except Exception as e: logger.error("Could not run command: %s" % str(e)) return json_response(error="Could not run command: %s" % str(e), status="ERROR", status_code=500)
def put_file(request, machine, path): """Writes the uploaded file to path and returns the path Keyword arguments: request -- HttpRequest containing the data machine_name -- name of the machine path -- path to file """ # Get data from request body data = request.read() # Write data to temporary location # TODO: Get temporary path from settings.py tmp_file = tempfile.NamedTemporaryFile(prefix="newt_") tmp_file.write(data) tmp_file.file.flush() src = "file:///%s" % tmp_file.name env = gridutil.get_cred_env(request.user) dest = gridutil.get_grid_path(machine, path) logger.debug("Putting file to location: %s" % dest) (output, error, retcode) = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-url-copy %s %s" % (src, dest), env=env) if retcode != 0: return json_response(content=output, status="ERROR", status_code=500, error=error) tmp_file.close() return {'location': path}
def get_info(request, machine_name, job_id): """Gets the information of a job, given the id Keyword arguments: machine_name -- name of the machine job_id -- the job id """ pass machine = slurmutil.GRID_RESOURCE_TABLE.get(machine_name, None) if not machine: return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name) env = slurmutil.get_cred_env(request.user) mycmd = "ssh " + machine["hostname"] + " ' " + ' sacct -j ' + job_id + " '" (output, error, retcode) = run_command(mycmd) if retcode != 0: return json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error) patt = re.compile( r'(?P<jobid>[^\s]+)\s+(?P<jobname>[^\s]+)\s+(?P<partition>[^\s]+)\s+(?P<account>[^\s]+)\s+(?P<alloccpus>[^\s]+)\s+(?P<state>[^\s]+)\s+(?P<exitcode>.*)$' ) output = output.splitlines() output = [x.strip() for x in output] output = filter(lambda line: patt.match(line), output) output = list(map(lambda x: patt.match(x).groupdict(), output))[2:] #print( output ) return (output)
def get_dir(request, machine_name, path): try: command = 'ls -la %s' % path (output, error, retcode) = run_command(command) if retcode != 0: return json_response(content=output, status="ERROR", status_code=500, error=error) # import pdb; pdb.set_trace() # Split the lines output = map(lambda i: i.strip(), output.split('\n')) # "awesome" regular expression that captures ls output of the form: # drwxrwxr-x 4 shreyas newt 32768 Apr 15 10:59 home patt=re.compile(r'(?P<perms>[\+\w@-]{10,})\s+(?P<hardlinks>\d+)\s+(?P<user>\S+)\s+(?P<group>\S+)\s+(?P<size>\d+)\s+(?P<date>\w{3}\s+\d+\s+[\d\:]+)\s+(?P<name>.+)$') # filter out stuff that doesn't match pattern output = filter(lambda line: patt.match(line), output) # Convert output into dict from group names output = map(lambda x: patt.match(x).groupdict(), output) for line in output: if line['perms'].startswith('l'): name, symlink = line['name'].split(' -> ') line['name'] = name line['symlink'] = symlink else: line['symlink'] = "" return output except Exception as e: logger.error("Could not get directory %s" % str(e)) return json_response(status="ERROR", status_code=500, error="Could not get directory: %s" % str(e))
def view_queue(request, machine_name): """Returns the current state of the queue in a list Keyword arguments: request -- Django HttpRequest machine_name -- name of the machine """ pass machine = slurmutil.GRID_RESOURCE_TABLE.get(machine_name, None) if not machine: return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name) env = slurmutil.get_cred_env(request.user) mycmd = "ssh " + machine["hostname"] + " ' " + machine["qstat"][ "bin"] + " '" (output, error, retcode) = run_command(mycmd) if retcode != 0: return json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error) patt = re.compile( r'(?P<jobid>[^\s]+)\s+(?P<partition>[^\s]+)\s+(?P<job_name>[^\s]+)\s+(?P<user>[^\s]+)\s+(?P<state>[^\s]+)\s+(?P<time>[^\s]+)\s+(?P<nodes>\d+)\s+(?P<nodelist>.*)$' ) output = output.splitlines() output = [x.strip() for x in output] output = filter(lambda line: patt.match(line), output) output = map(lambda x: patt.match(x).groupdict(), output) #print( list(output) ) return list(output)
def execute(request, machine_name, command): conf = settings.NEWT_CONFIG try: hostname = None for s in conf['SYSTEMS']: if machine_name == s['NAME']: hostname = s['HOSTNAME'] break if hostname is None: return json_response(status="ERROR", status_code=404, error="Unrecognized system: %s" % machine_name) #user = request.POST.get('sudo_user') logger.debug("Running command(ssh): %s (@ %s)" % (command, machine_name)) #command = "sudo -u %s %s " % (user, command) command = 'ssh %s " %s " ' % (hostname, command) (output, error, retcode) = run_command(command) response = {'output': output, 'error': error, 'retcode': retcode} return response except Exception as e: logger.error("Could not run command: %s" % str(e)) return json_response(error="Could not run command: %s" % str(e), status="ERROR", status_code=500)
def execute(request, machine_name, command): try: logger.debug("Running command: %s" % command) (output, error, retcode) = run_command(command) response = {"output": output, "error": error, "retcode": retcode} return response except Exception as e: logger.error("Could not run command: %s" % str(e)) return json_response(error="Could not run command: %s" % str(e), status="ERROR", status_code=500)
def submit_job(request, machine_name): """Submits a job to the queue Keyword arguments: request -- Django HttpRequest machine_name -- name of the machine """ pass machine = slurmutil.GRID_RESOURCE_TABLE.get(machine_name, None) if not machine: return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name) qsub = machine['qsub']['bin'] env = slurmutil.get_cred_env(request.user) user = request.user # User.objects.get(username=username) if request.POST.get("jobfile", False): # Create command for sbatch on an existing slurm file job_file_path = request.POST.get("jobfile") jobfile = job_file_path cmd = "%s %s" % (qsub, job_file_path) elif request.POST.get("jobscript", False): # Create command for qsub from stdin data job_script = request.POST.get("jobscript").encode() # Creates a temporary job file tmp_job_file = tempfile.NamedTemporaryFile( prefix="newt_", dir='/HOME/nscc-gz_jiangli/tmp', delete=False) print(job_script) tmp_job_file.write(job_script) tmp_job_file.flush() jobfile = tmp_job_file.name cmd = "%s %s" % (qsub, tmp_job_file.name) else: return json_response(status="ERROR", status_code=400, error="No data received") job = HPCJob(user=user, jobfile=jobfile, machine=machine_name) job.save() try: #runner = GlobusHelper(request.user) cmd_str = "ssh " + machine["hostname"] + ' " ' + cmd + ' " ' print(cmd_str) (output, error, retcode) = run_command(cmd_str, env=env) except Exception as ex: return json_response(status="ERROR", status_code=500, error="qsub failed with error: %s" % str(ex)) if retcode != 0: return json_response(status="ERROR", status_code=500, error="qsub failed with error: %s" % error) job.jobid = output.strip().split(' ')[-1] job.save() return {"jobid": job.jobid}
def delete_job_task_unsafty(self, taskenv, job_id): #mycmd = ' scancel ' + str( job_id ) bincommand = slurmutil.GRID_RESOURCE_TABLE.get( taskenv["machine"]).get('qdel')['bin'] mycmd = ' %s %s ' % (bincommand, str(job_id)) (output, error, retcode) = run_command(mycmd, bash=True) if retcode != 0: return worker_json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error) return worker_json_response(output)
def execute(request, machine_name, command): try: user = request.POST.get("sudo_user") logger.debug("Running command: %s as %s" % (command, user)) command = "sudo -u %s %s " % (user, command) (output, error, retcode) = run_command(command) response = {"output": output, "error": error, "retcode": retcode} return response except Exception as e: logger.error("Could not run command: %s" % str(e)) return json_response(error="Could not run command: %s" % str(e), status="ERROR", status_code=500)
def view_queue(request, machine_name): """Returns the current state of the queue in a list Keyword arguments: request -- Django HttpRequest machine_name -- name of the machine """ (output, error, retcode) = run_command("ps -aj") patt = re.compile(r'(?P<user>[^\s]+)\s+(?P<jobid>\d+)\s+(?P<ppid>\d+)\s+(?P<pgid>\d+)\s+(?P<sess>\d+)\s+(?P<jobc>\d+)\s+(?P<status>[^\s]+)\s+(?P<tt>[^\s]+)\s+(?P<timeuse>[^\s]+)\s+(?P<command>.+)') processes = output.splitlines()[1:] processes = map(lambda x: patt.match(x).groupdict(), processes) return processes
def delete_job(request, machine_name, job_id): """Gets the information of a job, given the id Keyword arguments: request -- Django HttpRequest machine_name -- name of the machine job_id -- the job id """ (output, error, retcode) = run_command("kill %s" % job_id) if retcode != 0: return json_response(status="ERROR", status_code=500, error=error) return {"output": output}
def execute(request, machine_name, command): try: user = request.POST.get('sudo_user') logger.debug("Running command: %s as %s" % (command, user)) command = "sudo -u %s %s " % (user, command) (output, error, retcode) = run_command(command) response = {'output': output, 'error': error, 'retcode': retcode} return response except Exception as e: logger.error("Could not run command: %s" % str(e)) return json_response(error="Could not run command: %s" % str(e), status="ERROR", status_code=500)
def delete_job(request, machine_name, job_id): machine = slurmutil.GRID_RESOURCE_TABLE.get(machine_name, None) if not machine: return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name) env = slurmutil.get_cred_env(request.user) mycmd = "ssh " + machine["hostname"] + " ' " + ' scancel ' + job_id + " '" (output, error, retcode) = run_command(mycmd) if retcode != 0: return json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error) return (output)
def view_queue_task_unsafty(self, taskenv): mycmd_conf = slurmutil.GRID_RESOURCE_TABLE.get( taskenv["machine"]).get('qstat') #mycmd = "ssh " + machine["hostname"] + " ' " + machine["qstat"]["bin"] + " '" (output, error, retcode) = run_command(mycmd_conf['bin']) if retcode != 0: return worker_json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error) df = str_to_dataframe(output, index=mycmd_conf.get('index', None), split=mycmd_conf.get('split', None), parser_cls=mycmd_conf.get('parser_cls', None)) return worker_json_response(json.loads(df.to_json()))
def put_file(request, machine, path): data = request.read() # Write data to temporary location # TODO: Get temporary path from settings.py tmp_file = tempfile.NamedTemporaryFile(prefix="newt_") tmp_file.write(data) tmp_file.file.flush() src = tmp_file.name dest = path (output, error, retcode) = run_command("cp %s %s" % (src, dest)) if retcode != 0: return json_response(content=output, status="ERROR", status_code=500, error=error) tmp_file.close() return {'location': path}
def view_queue(request, machine_name): """Returns the current state of the queue in a list Keyword arguments: request -- Django HttpRequest machine_name -- name of the machine """ (output, error, retcode) = run_command('ps -eo "%U %p %P %r %y %x %c %a"') #patt = re.compile(r'(?P<user>[^\s]+)\s+(?P<jobid>\d+)\s+(?P<ppid>\d+)\s+(?P<pgid>\d+)\s+(?P<sess>\d+)\s+(?P<jobc>\d+)\s+(?P<status>[^\s]+)\s+(?P<tt>[^\s]+)\s+(?P<timeuse>[^\s]+)\s+(?P<command>.+)') patt = re.compile( r'(?P<user>[^\s]+)\s+(?P<jobid>\d+)\s+(?P<ppid>\d+)\s+(?P<pgid>\d+)\s+(?P<tty>[^\s]+)\s+(?P<time>[^\s]+)(?P<command>.+)' ) processes = output.splitlines()[1:] #print( processes ) processes = list(map(lambda x: patt.match(x).groupdict(), processes)) return processes
def get_dir(request, machine_name, path): """Returns a directory listing of path (as an array) Keyword arguments: machine_name -- name of the machine path -- path to file """ try: env = gridutil.get_cred_env(request.user) path = gridutil.get_grid_path(machine_name, path) output, error, retcode = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/uberftp -ls %s" % path, env=env) if retcode != 0: return json_response(content=output, status="ERROR", status_code=500, error=error) # Split the lines output = map(lambda i: i.strip(), output.splitlines()) # regular expression that captures ls output of the form: # drwxrwxr-x 4 shreyas newt 32768 Apr 15 10:59 home patt = re.compile( r'(?P<perms>[\+\w@-]{10,})\s+(?P<hardlinks>\d+)\s+(?P<user>\S+)\s+(?P<group>\S+)\s+(?P<size>\d+)\s+(?P<date>\w{3}\s+\d+\s+[\d\:]+)\s+(?P<name>.+)$' ) # filter out stuff that doesn't match pattern output = filter(lambda line: patt.match(line), output) # break up line into tuple: (perms, hl, user, group, size, date, filename) output = map(lambda x: patt.match(x).groupdict(), output) for line in output: if line['perms'].startswith('l'): name, symlink = line['name'].split(' -> ') line['name'] = name line['symlink'] = symlink else: line['symlink'] = "" return output except Exception as e: logger.error("Could not get directory %s" % str(e)) return json_response(status="ERROR", status_code=500, error="Could not get directory: %s" % str(e))
def execute(request, machine_name, command): conf = settings.NEWT_CONFIG for s in conf['SYSTEMS']: if machine_name == s['NAME']: hostname = s['HOSTNAME'] break try: (output, error, retcode) = run_command(command,hostname) response = { 'output': output, 'error': error, 'retcode': retcode } return response except Exception as e: logger.error("Could not run command: %s" % str(e)) return json_response(error="Could not run command: %s" % str(e), status="ERROR", status_code=500)
def get_info_task_unsafty(self, taskenv, HPCJobid): job = HPCJob.objects.get(id=HPCJobid) job_id = job.jobid command_conf = slurmutil.GRID_RESOURCE_TABLE.get( taskenv["machine"]).get('qjobstat') mycmd = '%s %s' % (command_conf['bin'], job_id ) #' sacct -lPj ' + str(job_id ) (output, error, retcode) = run_command(mycmd, bash=True) if retcode != 0: return worker_json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error) df = str_to_dataframe(output, split=command_conf.get('split', None), index=command_conf.get('index', None), parser_cls=command_conf.get('parser_cls', None)) return worker_json_response(content=json.loads(df.to_json()))
def view_queue_task_unsafty(self, taskenv): """ Celery task for view_queue to call""" #mycmd = machine["qstat"]["bin"] mycmd = "/usr/bin/squeue" (output, error, retcode) = run_command(mycmd) if retcode != 0: return worker_json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error) patt = re.compile( r'(?P<jobid>[^\s]+)\s+(?P<partition>[^\s]+)\s+(?P<job_name>[^\s]+)\s+(?P<user>[^\s]+)\s+(?P<state>[^\s]+)\s+(?P<time>[^\s]+)\s+(?P<nodes>\d+)\s+(?P<nodelist>.*)$' ) output = output.splitlines() output = [x.strip() for x in output] output = filter(lambda line: patt.match(line), output) output = map(lambda x: patt.match(x).groupdict(), output) #print( list(output) ) return worker_json_response(list(output))
def execute_task_unsafy(self, task_env, command): try: (output, error, retcode) = run_command(command, bash=True) response = { "content": { 'output': output, 'error': error, 'retcode': retcode } } return response except Exception as e: #return json_response(error="Could not run command: %s" % str(e), status="ERROR", status_code=500) return { "error": "Could not run command: %s" % str(e), "status": "ERROR", "status_code": 500 }
def get_info(request, machine_name, job_id): """Gets the information of a job, given the id Keyword arguments: machine_name -- name of the machine job_id -- the job id """ pass machine = slurmutil.GRID_RESOURCE_TABLE.get(machine_name, None) if not machine: return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name) env = slurmutil.get_cred_env(request.user) mycmd = "ssh " + machine["hostname"] + " ' " + ' sacct -j ' + job_id + " '" job = HPCJob.objects.get(machine=machine_name, jobid=job_id) if job.state == "COMPLETED" or job.state == "FAILED": return { "partition": job.state, "jobid": job.jobid, "state": job.state, "exitcode": job.exit_code, "jobname": job.job_name } (output, error, retcode) = run_command(mycmd) if retcode != 0: return json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error) patt = re.compile( r'(?P<jobid>[^\s]+)\s+(?P<jobname>[^\s]+)\s+(?P<partition>[^\s]+)\s+(?P<account>[^\s]+)\s+(?P<alloccpus>[^\s]+)\s+(?P<state>[^\s]+)\s+(?P<exitcode>.*)$' ) output = output.splitlines() output = [x.strip() for x in output] output = filter(lambda line: patt.match(line), output) output = list(map(lambda x: patt.match(x).groupdict(), output))[2:] #print( output ) # {"partition": "work", "account": "nscc-gz", "alloccpus": "24", "jobid": "3422542", "state": "COMPLETED", "exitcode": "0:0", "jobname": "newt_cs0r+"} job.partition = output[0]["partition"] job.exit_code = output[0]["exitcode"].split(':')[1] job.job_name = output[0]["jobname"] job.state = output[0]["state"] job.save() return (output)
def get_dir(request, machine_name, path): try: command = 'ls -la --time-style=long-iso %s' % path (output, error, retcode) = run_command(command) if retcode != 0: return json_response(content=output, status="ERROR", status_code=500, error=error) # import pdb; pdb.set_trace() # Split the lines output = list(map(lambda i: i.strip(), output.split('\n'))) print('OUT1', output) # "awesome" regular expression that captures ls output of the form: # drwxrwxr-x 4 shreyas newt 32768 Apr 15 10:59 home #patt=re.compile(r'(?P<perms>[\+\w@-]{10,})\s+(?P<hardlinks>\d+)\s+(?P<user>\S+)\s+(?P<group>\S+)\s+(?P<size>\d+)\s+(?P<date>\w{3}\s+\d+\s+[\d\:]+)\s+(?P<name>.+)$') patt = re.compile( r'(?P<perms>[\+\w@-]{10,}\.?)\s+(?P<hardlinks>\d+)\s+(?P<user>\S+)\s+(?P<group>\S+)\s+(?P<size>\d+)\s+(?P<date>[\d\-]{10}\s[\d\:]+)\s+(?P<name>.+)$' ) # filter out stuff that doesn't match pattern output = list(filter(lambda line: patt.match(line), output)) print('OUT2', (output)) # Convert output into dict from group names output = list(map(lambda x: patt.match(x).groupdict(), output)) print('OUT3 :', output) for line in output: if line['perms'].startswith('l'): name, symlink = line['name'].split(' -> ') line['name'] = name line['symlink'] = symlink else: line['symlink'] = "" return list(output) except Exception as e: logger.error("Could not get directory %s" % str(e)) return json_response(status="ERROR", status_code=500, error="Could not get directory: %s" % str(e))
def put_file_task_unsafty(self, task_env, temphost, src, dest): # safy task . cp , ... , rm tmp file , return cookie_file = os.path.join(getpwuid(os.getuid()).pw_dir, localcookies) command_src = ''' curl -X GET -s -b %s "%s://%s:%s/api/file/%s/%s?&download=True" -o %s ''' % ( cookie_file, conf_connection['protocol'], temphost, conf_connection['port'], machine_default, src, dest) #(output, error, retcode) = run_command(" scp %s:%s %s " % ( temphost, src, dest)) (output, error, retcode) = run_command(command_src, bash=True) if retcode != 0: #return json_response(content=output, status="ERROR", status_code=500, error=error) return { 'content': output, 'status': "ERROR", 'status_code': 500, 'error': error } #(output, error, retcode) = run_command(" ssh %s rm %s " % ( temphost, src)) #if retcode != 0: # return json_response(content=output, status="ERROR", status_code=500, error=error) return {'content': {'location': dest}} pass
def get_dir(request, machine_name, path): """Returns a directory listing of path (as an array) Keyword arguments: machine_name -- name of the machine path -- path to file """ try: env = gridutil.get_cred_env(request.user) path = gridutil.get_grid_path(machine_name, path) output, error, retcode = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/uberftp -ls %s" % path, env=env) if retcode != 0: return json_response(content=output, status="ERROR", status_code=500, error=error) # Split the lines output = map(lambda i: i.strip(), output.splitlines()) # regular expression that captures ls output of the form: # drwxrwxr-x 4 shreyas newt 32768 Apr 15 10:59 home patt=re.compile(r'(?P<perms>[\+\w@-]{10,})\s+(?P<hardlinks>\d+)\s+(?P<user>\S+)\s+(?P<group>\S+)\s+(?P<size>\d+)\s+(?P<date>\w{3}\s+\d+\s+[\d\:]+)\s+(?P<name>.+)$') # filter out stuff that doesn't match pattern output = filter(lambda line: patt.match(line), output) # break up line into tuple: (perms, hl, user, group, size, date, filename) output = map(lambda x: patt.match(x).groupdict(), output) for line in output: if line['perms'].startswith('l'): name, symlink = line['name'].split(' -> ') line['name'] = name line['symlink'] = symlink else: line['symlink'] = "" return output except Exception as e: logger.error("Could not get directory %s" % str(e)) return json_response(status="ERROR", status_code=500, error="Could not get directory: %s" % str(e))
def download_path(request, machine_name, path): """Returns a StreamingHttpResponse with the file Keyword arguments: machine_name -- name of the machine path -- path to file """ src = gridutil.get_grid_path(machine_name, path) env = gridutil.get_cred_env(request.user) tmp_file = tempfile.NamedTemporaryFile(prefix="newt_") dest = "file://"+tmp_file.name logger.debug("File download requested: %s (%s)" % (path, src)) (output, error, retcode) = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-url-copy %s %s" % (src, dest), env=env) if retcode != 0: logger.warning("Unable to download file: %s" % src) return json_response(content=output, status="ERROR", status_code=500, error=error) # filename = path.rsplit("/")[-1] # f = open(dest+"filename", "r") mimetype = mimetypes.guess_type(tmp_file.name) if mimetype is None: mimetype = "application/octet-stream" return StreamingHttpResponse(tmp_file, content_type=mimetype)
def download_path_task_unsafty(self, taskenv, path): if path.startswith('/~'): path = os.path.join(getpwuid(os.getuid()).pw_dir, path[3:]) src = path temphost = taskenv["host"] #dest = '/tmp/tmpfile/'+ self.request.id cookie_file = os.path.join(getpwuid(os.getuid()).pw_dir, localcookies) command = ''' curl -b %s -T %s "%s://%s:%s/api/file/%s/%s?local=True" ''' % ( cookie_file, src, conf_connection['protocol'], temphost, conf_connection['port'], machine_default, self.request.id) #(output, error, retcode) = run_command(" scp %s %s:%s " % ( src , temphost, dest)) (output, error, retcode) = run_command(command, bash=True) print(output, error, retcode, command) if retcode != 0: return { "content": output, "status": "ERROR", "status_code": 500, "error": error, } #return json_response(content=output, status="ERROR", status_code=500, error=error) return self.request.id pass
def view_queue(request, machine_name): """Returns the current state of the queue in a list Keyword arguments: request -- Django HttpRequest machine_name -- name of the machine """ machine = gridutil.GRID_RESOURCE_TABLE.get(machine_name, None) if not machine: return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name) env = gridutil.get_cred_env(request.user) (output, error, retcode) = run_command( gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-job-run %s /project/projectdirs/osp/newt_tools/qs_moab.sh" % (machine['hostname']), env=env) patt = re.compile( r'(?P<jobid>[^\s]+)\s+(?P<status>[^\s]+)\s+(?P<user>[^\s]+)\s+(?P<job_name>[^\s]+)\s+(?P<nodes>\d+)\s+(?P<walltime>[^\s]+)\s+(?P<time_use>[^\s]+)\s+(?P<time_submit>\w{3}\s\d{1,2}\s[\d\:]+)\s+(?P<rank>[^\s]+)\s+(?P<queue>[^\s]+)\s+(?P<q_state>[^\s]+)\s+(?P<processors>[^\s]+)\s*(?P<details>.*)$' ) if retcode != 0: return json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error) # filter out stuff that doesn't match pattern output = output.splitlines() output = [x.strip() for x in output] output = filter(lambda line: patt.match(line), output) # Convert output into dict from group names output = map(lambda x: patt.match(x).groupdict(), output) return output
def run_job(self, command, host, flags=""): cmd_str = self.GLOBUS_JOB_RUN_BIN + " %s %s %s" % (host, flags, command) return run_command(cmd_str, env=self.env)