def put_file(request, machine, path):
"""Writes the uploaded file to path and returns the path
Keyword arguments:
request -- HttpRequest containing the data
machine_name -- name of the machine
path -- path to file
"""
# Get data from request body
data = request.read()
# Write data to temporary location
# TODO: Get temporary path from settings.py
tmp_file = tempfile.NamedTemporaryFile(prefix="newt_")
tmp_file.write(data)
tmp_file.file.flush()
src = "file:///%s" % tmp_file.name
env = gridutil.get_cred_env(request.user)
dest = gridutil.get_grid_path(machine, path)
logger.debug("Putting file to location: %s" % dest)
(output, error, retcode) = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-url-copy %s %s" % (src, dest), env=env)
if retcode != 0:
return json_response(content=output, status="ERROR", status_code=500, error=error)
tmp_file.close()
return {'location': path}
def execute(request, machine_name, command):
"""Returns a the result of running command on machine_name
Keyword arguments:
machine_name -- name of the machine
command -- command to run
"""
machine = gridutil.GRID_RESOURCE_TABLE.get(machine_name, None)
if not machine:
return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name)
# Get the grid credentials for the user
env = gridutil.get_cred_env(request.user)
try:
# Run the command using globus-job-run
(output, error, retcode) = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-job-run %s %s" % (machine['hostname'], command), env=env)
response = {
"output": output,
"error": error,
"retcode": retcode
}
return response
except Exception as e:
logger.error("Could not run command: %s" % str(e))
return json_response(error="Could not run command: %s" % str(e), status="ERROR", status_code=500)
def execute(request, machine_name, command):
"""Returns a the result of running command on machine_name
Keyword arguments:
machine_name -- name of the machine
command -- command to run
"""
machine = gridutil.GRID_RESOURCE_TABLE.get(machine_name, None)
if not machine:
return json_response(status="ERROR",
status_code=400,
error="Invalid machine name: %s" % machine_name)
# Get the grid credentials for the user
env = gridutil.get_cred_env(request.user)
try:
# Run the command using globus-job-run
(output, error, retcode) = run_command(
gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-job-run %s %s" %
(machine['hostname'], command),
env=env)
response = {"output": output, "error": error, "retcode": retcode}
return response
except Exception as e:
logger.error("Could not run command: %s" % str(e))
return json_response(error="Could not run command: %s" % str(e),
status="ERROR",
status_code=500)
def view_queue(request, machine_name):
"""Returns the current state of the queue in a list
Keyword arguments:
request -- Django HttpRequest
machine_name -- name of the machine
"""
machine = gridutil.GRID_RESOURCE_TABLE.get(machine_name, None)
if not machine:
return json_response(status="ERROR", status_code=400, error="Invalid machine name: %s" % machine_name)
env = gridutil.get_cred_env(request.user)
(output, error, retcode) = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-job-run %s /project/projectdirs/osp/newt_tools/qs_moab.sh" % (machine['hostname']), env=env)
patt = re.compile(r'(?P<jobid>[^\s]+)\s+(?P<status>[^\s]+)\s+(?P<user>[^\s]+)\s+(?P<job_name>[^\s]+)\s+(?P<nodes>\d+)\s+(?P<walltime>[^\s]+)\s+(?P<time_use>[^\s]+)\s+(?P<time_submit>\w{3}\s\d{1,2}\s[\d\:]+)\s+(?P<rank>[^\s]+)\s+(?P<queue>[^\s]+)\s+(?P<q_state>[^\s]+)\s+(?P<processors>[^\s]+)\s*(?P<details>.*)$')
if retcode != 0:
return json_response(status="ERROR", status_code=500, error="Unable to get queue: %s" % error)
# filter out stuff that doesn't match pattern
output = output.splitlines()
output = [x.strip() for x in output]
output = filter(lambda line: patt.match(line), output)
# Convert output into dict from group names
output = map(lambda x: patt.match(x).groupdict(), output)
return output
def download_path(request, machine_name, path):
"""Returns a StreamingHttpResponse with the file
Keyword arguments:
machine_name -- name of the machine
path -- path to file
"""
src = gridutil.get_grid_path(machine_name, path)
env = gridutil.get_cred_env(request.user)
tmp_file = tempfile.NamedTemporaryFile(prefix="newt_")
dest = "file://" + tmp_file.name
logger.debug("File download requested: %s (%s)" % (path, src))
(output, error,
retcode) = run_command(gridutil.GLOBUS_CONF['LOCATION'] +
"bin/globus-url-copy %s %s" % (src, dest),
env=env)
if retcode != 0:
logger.warning("Unable to download file: %s" % src)
return json_response(content=output,
status="ERROR",
status_code=500,
error=error)
# filename = path.rsplit("/")[-1]
# f = open(dest+"filename", "r")
mimetype = mimetypes.guess_type(tmp_file.name)
if mimetype is None:
mimetype = "application/octet-stream"
return StreamingHttpResponse(tmp_file, content_type=mimetype)
def get_dir(request, machine_name, path):
"""Returns a directory listing of path (as an array)
Keyword arguments:
machine_name -- name of the machine
path -- path to file
"""
try:
env = gridutil.get_cred_env(request.user)
path = gridutil.get_grid_path(machine_name, path)
output, error, retcode = run_command(gridutil.GLOBUS_CONF['LOCATION'] +
"bin/uberftp -ls %s" % path,
env=env)
if retcode != 0:
return json_response(content=output,
status="ERROR",
status_code=500,
error=error)
# Split the lines
output = map(lambda i: i.strip(), output.splitlines())
# regular expression that captures ls output of the form:
# drwxrwxr-x 4 shreyas newt 32768 Apr 15 10:59 home
patt = re.compile(
r'(?P<perms>[\+\w@-]{10,})\s+(?P<hardlinks>\d+)\s+(?P<user>\S+)\s+(?P<group>\S+)\s+(?P<size>\d+)\s+(?P<date>\w{3}\s+\d+\s+[\d\:]+)\s+(?P<name>.+)$'
)
# filter out stuff that doesn't match pattern
output = filter(lambda line: patt.match(line), output)
# break up line into tuple: (perms, hl, user, group, size, date, filename)
output = map(lambda x: patt.match(x).groupdict(), output)
for line in output:
if line['perms'].startswith('l'):
name, symlink = line['name'].split(' -> ')
line['name'] = name
line['symlink'] = symlink
else:
line['symlink'] = ""
return output
except Exception as e:
logger.error("Could not get directory %s" % str(e))
return json_response(status="ERROR",
status_code=500,
error="Could not get directory: %s" % str(e))
def get_dir(request, machine_name, path):
"""Returns a directory listing of path (as an array)
Keyword arguments:
machine_name -- name of the machine
path -- path to file
"""
try:
env = gridutil.get_cred_env(request.user)
path = gridutil.get_grid_path(machine_name, path)
output, error, retcode = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/uberftp -ls %s" % path,
env=env)
if retcode != 0:
return json_response(content=output, status="ERROR", status_code=500, error=error)
# Split the lines
output = map(lambda i: i.strip(), output.splitlines())
# regular expression that captures ls output of the form:
# drwxrwxr-x 4 shreyas newt 32768 Apr 15 10:59 home
patt=re.compile(r'(?P<perms>[\+\w@-]{10,})\s+(?P<hardlinks>\d+)\s+(?P<user>\S+)\s+(?P<group>\S+)\s+(?P<size>\d+)\s+(?P<date>\w{3}\s+\d+\s+[\d\:]+)\s+(?P<name>.+)$')
# filter out stuff that doesn't match pattern
output = filter(lambda line: patt.match(line), output)
# break up line into tuple: (perms, hl, user, group, size, date, filename)
output = map(lambda x: patt.match(x).groupdict(), output)
for line in output:
if line['perms'].startswith('l'):
name, symlink = line['name'].split(' -> ')
line['name'] = name
line['symlink'] = symlink
else:
line['symlink'] = ""
return output
except Exception as e:
logger.error("Could not get directory %s" % str(e))
return json_response(status="ERROR", status_code=500, error="Could not get directory: %s" % str(e))
def download_path(request, machine_name, path):
"""Returns a StreamingHttpResponse with the file
Keyword arguments:
machine_name -- name of the machine
path -- path to file
"""
src = gridutil.get_grid_path(machine_name, path)
env = gridutil.get_cred_env(request.user)
tmp_file = tempfile.NamedTemporaryFile(prefix="newt_")
dest = "file://"+tmp_file.name
logger.debug("File download requested: %s (%s)" % (path, src))
(output, error, retcode) = run_command(gridutil.GLOBUS_CONF['LOCATION'] + "bin/globus-url-copy %s %s" % (src, dest), env=env)
if retcode != 0:
logger.warning("Unable to download file: %s" % src)
return json_response(content=output, status="ERROR", status_code=500, error=error)
# filename = path.rsplit("/")[-1]
# f = open(dest+"filename", "r")
mimetype = mimetypes.guess_type(tmp_file.name)
if mimetype is None:
mimetype = "application/octet-stream"
return StreamingHttpResponse(tmp_file, content_type=mimetype)
def view_queue(request, machine_name):
"""Returns the current state of the queue in a list
Keyword arguments:
request -- Django HttpRequest
machine_name -- name of the machine
"""
machine = gridutil.GRID_RESOURCE_TABLE.get(machine_name, None)
if not machine:
return json_response(status="ERROR",
status_code=400,
error="Invalid machine name: %s" % machine_name)
env = gridutil.get_cred_env(request.user)
(output, error, retcode) = run_command(
gridutil.GLOBUS_CONF['LOCATION'] +
"bin/globus-job-run %s /project/projectdirs/osp/newt_tools/qs_moab.sh"
% (machine['hostname']),
env=env)
patt = re.compile(
r'(?P<jobid>[^\s]+)\s+(?P<status>[^\s]+)\s+(?P<user>[^\s]+)\s+(?P<job_name>[^\s]+)\s+(?P<nodes>\d+)\s+(?P<walltime>[^\s]+)\s+(?P<time_use>[^\s]+)\s+(?P<time_submit>\w{3}\s\d{1,2}\s[\d\:]+)\s+(?P<rank>[^\s]+)\s+(?P<queue>[^\s]+)\s+(?P<q_state>[^\s]+)\s+(?P<processors>[^\s]+)\s*(?P<details>.*)$'
)
if retcode != 0:
return json_response(status="ERROR",
status_code=500,
error="Unable to get queue: %s" % error)
# filter out stuff that doesn't match pattern
output = output.splitlines()
output = [x.strip() for x in output]
output = filter(lambda line: patt.match(line), output)
# Convert output into dict from group names
output = map(lambda x: patt.match(x).groupdict(), output)
return output
