def symlink():
auth_header = request.headers[AUTH_HEADER_NAME]
try:
system_name = request.headers["X-Machine-Name"]
except KeyError as e:
app.logger.error("No machinename given")
return jsonify(description="No machine name given"), 400
# PUBLIC endpoints from Kong to users
if system_name not in SYSTEMS_PUBLIC:
header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
return jsonify(description="Failed to create symlink",
error="Machine does not exist"), 400, header
# select index in the list corresponding with machine name
system_idx = SYSTEMS_PUBLIC.index(system_name)
system_addr = SYS_INTERNALS[system_idx]
try:
linkPath = request.form["linkPath"]
if linkPath == "":
return jsonify(description="Failed to create symlink",
error="'linkPath' value is empty"), 400
except BadRequestKeyError:
return jsonify(description="Failed to create symlink",
error="'linkPath' query string missing"), 400
try:
targetPath = request.form["targetPath"]
if targetPath == "":
return jsonify(description="Failed to create symlink",
error="'targetPath' value is empty"), 400
except BadRequestKeyError:
return jsonify(description="Failed to create symlink",
error="'targetPath' query string missing"), 400
action = f"timeout {UTILITIES_TIMEOUT} ln -s -- '{targetPath}' '{linkPath}'"
retval = exec_remote_command(auth_header, system_name, system_addr, action)
if retval["error"] != 0:
error_str = retval["msg"]
error_code = retval["error"]
service_msg = "Failed to create symlink"
ret_data = check_command_error(error_str, error_code, service_msg)
# if generic "error" not in the dict
try:
jsonify(description=ret_data["description"],
error=ret_data["error"]
), ret_data["status_code"], ret_data["header"]
except:
return jsonify(description=ret_data["description"]
), ret_data["status_code"], ret_data["header"]
return jsonify(description="Success create the symlink"), 201
def checksum():
auth_header = request.headers[AUTH_HEADER_NAME]
try:
system_name = request.headers["X-Machine-Name"]
except KeyError as e:
app.logger.error("No machinename given")
return jsonify(description="No machine name given"), 400
# PUBLIC endpoints from Kong to users
if system_name not in SYSTEMS_PUBLIC:
header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
return jsonify(description="Error obatining checksum",
error="Machine does not exist"), 400, header
# select index in the list corresponding with machine name
system_idx = SYSTEMS_PUBLIC.index(system_name)
system_addr = SYS_INTERNALS[system_idx]
try:
path = request.args.get("targetPath")
if path == "":
return jsonify(description="Error obatining checksum",
error="'targetPath' value is empty"), 400
except BadRequestKeyError:
return jsonify(description="Error obatining checksum",
error="'targetPath' query string missing"), 400
action = f"timeout {UTILITIES_TIMEOUT} sha256sum -- '{path}'"
retval = exec_remote_command(auth_header, system_name, system_addr, action)
if retval["error"] != 0:
error_str = retval["msg"]
error_code = retval["error"]
service_msg = "Error obtaining checksum"
ret_data = check_command_error(error_str, error_code, service_msg)
# if generic "error" not in the dict
try:
jsonify(description=ret_data["description"],
error=ret_data["error"]
), ret_data["status_code"], ret_data["header"]
except:
return jsonify(description=ret_data["description"]
), ret_data["status_code"], ret_data["header"]
# on success: retval["msg"] = "checksum /path/to/file"
output = retval["msg"].split()[0]
return jsonify(description="Checksum successfully retrieved",
output=output), 200
def rm():
auth_header = request.headers[AUTH_HEADER_NAME]
try:
system_name = request.headers["X-Machine-Name"]
except KeyError as e:
app.logger.error("No machinename given")
return jsonify(description="No machine name given"), 400
# PUBLIC endpoints from Kong to users
if system_name not in SYSTEMS_PUBLIC:
header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
return jsonify(description="Error on delete operation",
error="Machine does not exist"), 400, header
# select index in the list corresponding with machine name
system_idx = SYSTEMS_PUBLIC.index(system_name)
system_addr = SYS_INTERNALS[system_idx]
try:
path = request.form["targetPath"]
if path == "":
return jsonify(description="Error on delete operation",
error="'targetPath' value is empty"), 400
except BadRequestKeyError:
return jsonify(description="Error on delete operation",
error="'targetPath' query string missing"), 400
# action to execute
# -r is for recursivelly delete files into directories
action = f"timeout {UTILITIES_TIMEOUT} rm -r --interactive=never -- '{path}'"
retval = exec_remote_command(auth_header, system_name, system_addr, action)
if retval["error"] != 0:
error_str = retval["msg"]
error_code = retval["error"]
service_msg = "Error on delete operation"
ret_data = check_command_error(error_str, error_code, service_msg)
# if generic "error" not in the dict
try:
jsonify(description=ret_data["description"],
error=ret_data["error"]
), ret_data["status_code"], ret_data["header"]
except:
return jsonify(description=ret_data["description"]
), ret_data["status_code"], ret_data["header"]
return jsonify(description="Success to delete file or directory.",
output=""), 204
def common_fs_operation(request, command):
try:
system_name = request.headers["X-Machine-Name"]
except KeyError as e:
app.logger.error("No machinename given")
return jsonify(description="No machine name given"), 400
# PUBLIC endpoints from Kong to users
if system_name not in SYSTEMS_PUBLIC:
header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
return jsonify(description=f"Error on {command} operation",
error="Machine does not exist"), 400, header
# select index in the list corresponding with machine name
system_idx = SYSTEMS_PUBLIC.index(system_name)
system_addr = SYS_INTERNALS[system_idx]
# get targetPath to apply command
tn = 'targetPath'
if request.method == 'GET':
targetPath = request.args.get("targetPath", None)
if (targetPath == None) and (command in ['base64', 'stat']):
# TODO: review API
tn = "sourcePath"
targetPath = request.args.get("sourcePath", None)
else: # DELETE, POST, PUT
targetPath = request.form.get("targetPath", None)
v = validate_input(targetPath)
if v != "":
return jsonify(description=f"Error on {command} operation",
error=f"'{tn}' {v}"), 400
if command in ['copy', 'rename']:
sourcePath = request.form.get("sourcePath", None)
v = validate_input(sourcePath)
if v != "":
return jsonify(description=f"Error on {command} operation",
error=f"'sourcePath' {v}"), 400
file_content = None
file_transfer = None
success_code = 200
if command == "base64":
action = f"base64 --wrap=0 -- '{targetPath}'"
file_transfer = 'download'
elif command == "checksum":
action = f"sha256sum -- '{targetPath}'"
elif command == "chmod":
mode = request.form.get("mode", None)
v = validate_input(mode)
if v != "":
return jsonify(description="Error on chmod operation",
error=f"'mode' {v}"), 400
action = f"chmod -v '{mode}' -- '{targetPath}'"
elif command == "chown":
owner = request.form.get("owner", "")
group = request.form.get("group", "")
if owner == "" and group == "":
return jsonify(description="Error in chown operation",
error="group or owner must be set"), 400
v = validate_input(owner + group)
if v != "":
return jsonify(description="Error in chown operation",
error=f"group or owner {v}"), 400
action = f"chown -v '{owner}':'{group}' -- '{targetPath}'"
elif command == "copy":
# -r is for recursivelly copy files into directories
action = f"cp --force -dR --preserve=all -- '{sourcePath}' '{targetPath}'"
success_code = 201
elif command == "file":
# -b: do not prepend filenames to output lines
action = f"file -b -- '{targetPath}'"
elif command == "head":
action = f"head -c {MAX_FILE_SIZE_BYTES} -- '{targetPath}'"
file_transfer = 'download'
elif command == "ls":
# if set shows entrys starting with . (not including . and/or .. dirs)
showhidden = request.args.get("showhidden", None)
showall = ""
if showhidden != None:
showall = "-A"
action = f"ls -l {showall} --time-style=+%Y-%m-%dT%H:%M:%S -- '{targetPath}'"
elif command == "mkdir":
try:
p = request.form["p"]
parent = "-p"
except BadRequestKeyError:
parent = ""
action = f"mkdir {parent} -- '{targetPath}'"
success_code = 201
elif command == "rename":
action = f"mv --force -- '{sourcePath}' '{targetPath}'"
elif command == "rm":
# -r is for recursivelly delete files into directories
action = f"rm -r --interactive=never -- '{targetPath}'"
success_code = 204
elif command == "stat":
action = f"stat --dereference -c %s -- '{targetPath}'"
elif command == "symlink":
linkPath = request.form.get("linkPath", None)
v = validate_input(linkPath)
if v != "":
return jsonify(description="Failed to create symlink",
error=f"'linkPath' value {v}"), 400
action = f"ln -s -- '{targetPath}' '{linkPath}'"
success_code = 201
elif command == "upload":
try:
if 'file' not in request.files:
return jsonify(description="Failed to upload file",
error="No file in query"), 400
file = request.files['file']
app.logger.info(f"Upload length: {file.content_length}")
v = validate_input(file.filename)
if v != "":
return jsonify(description="Failed to upload file",
error=f"Filename {v}"), 400
except:
return jsonify(description='Error on upload operation',
output=''), 400
filename = secure_filename(file.filename)
action = f"cat > '{targetPath}/{filename}'"
file_content = file.read()
file_transfer = 'upload'
success_code = 201
else:
app.logger.error(f"Unknown command on common_fs_operation: {command}")
return jsonify(description="Error on internal operation",
error="Internal error"), 400
[headers, ID] = get_tracing_headers(request)
action = f"ID={ID} timeout {UTILITIES_TIMEOUT} {action}"
retval = exec_remote_command(headers, system_name, system_addr, action,
file_transfer, file_content)
if retval["error"] != 0:
error_str = retval["msg"]
error_code = retval["error"]
service_msg = f"Error on {command} operation"
ret_data = check_command_error(error_str, error_code, service_msg)
# if generic "error" not in the dict
try:
return jsonify(description=ret_data["description"],
error=ret_data["error"]
), ret_data["status_code"], ret_data["header"]
except:
return jsonify(description=ret_data["description"]
), ret_data["status_code"], ret_data["header"]
description = f"Success to {command} file or directory."
output = ''
if command == 'checksum':
# return only hash, msg sintax: hash filename
output = retval["msg"].split()[0]
elif command in ['base64', 'chmod', 'chown', 'file', 'head', 'stat']:
output = retval["msg"]
elif command == 'ls':
description = "List of contents"
output = ls_parse(request, retval)
elif command == "upload":
description = "File upload successful"
return jsonify(description=description, output=output), success_code
def common_operation(request, command, method):
auth_header = request.headers[AUTH_HEADER_NAME]
try:
system_name = request.headers["X-Machine-Name"]
except KeyError as e:
app.logger.error("No machinename given")
return jsonify(description="No machine name given"), 400
# PUBLIC endpoints from Kong to users
if system_name not in SYSTEMS_PUBLIC:
header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
return jsonify(description="Error on " + command + " operation",
error="Machine does not exist"), 400, header
# select index in the list corresponding with machine name
system_idx = SYSTEMS_PUBLIC.index(system_name)
system_addr = SYS_INTERNALS[system_idx]
try:
sourcePath = request.form["sourcePath"]
if sourcePath == "":
return jsonify(description="Error on " + command + " operation",
error="'sourcePath' value is empty"), 400
except BadRequestKeyError:
return jsonify(description="Error on " + command + " operation",
error="'sourcePath' query string missing"), 400
try:
targetPath = request.form["targetPath"]
if targetPath == "":
return jsonify(description="Error on " + command + " operation",
error="'targetPath' value is empty"), 400
except BadRequestKeyError:
return jsonify(description="Error on " + command + " operation",
error="target query string missing"), 400
if command == "copy":
# action to execute
# -r is for recursivelly copy files into directories
action = f"timeout {UTILITIES_TIMEOUT} cp --force -dR --preserve=all -- '{sourcePath}' '{targetPath}'"
success_code = 201
elif command == "rename":
action = f"timeout {UTILITIES_TIMEOUT} mv --force -- '{sourcePath}' '{targetPath}'"
success_code = 200
else:
app.logger.error("Unknown command on common_operation: " + command)
return jsonify(description="Error on unkownon operation",
error="Unknown"), 400
retval = exec_remote_command(auth_header, system_name, system_addr, action)
if retval["error"] != 0:
error_str = retval["msg"]
error_code = retval["error"]
service_msg = f"Error on {command} operation"
ret_data = check_command_error(error_str, error_code, service_msg)
# if generic "error" not in the dict
try:
jsonify(description=ret_data["description"],
error=ret_data["error"]
), ret_data["status_code"], ret_data["header"]
except:
return jsonify(description=ret_data["description"]
), ret_data["status_code"], ret_data["header"]
return jsonify(description="Success to " + command + " file or directory.",
output=""), success_code
def view():
auth_header = request.headers[AUTH_HEADER_NAME]
try:
system_name = request.headers["X-Machine-Name"]
except KeyError as e:
app.logger.error("No machinename given")
return jsonify(description="No machine name given"), 400
# PUBLIC endpoints from Kong to users
if system_name not in SYSTEMS_PUBLIC:
header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
return jsonify(description="Failed to view file content",
error="Machine does not exist"), 400, header
# select index in the list corresponding with machine name
system_idx = SYSTEMS_PUBLIC.index(system_name)
system_addr = SYS_INTERNALS[system_idx]
path = request.args.get("targetPath")
if path == None:
return jsonify(description="Failed to view file content",
error="'targetPath' query string missing"), 400
if path == "":
return jsonify(description="Failed to view file content",
error="'targetPath' value is empty"), 400
# check file size
action = f"timeout {UTILITIES_TIMEOUT} stat --dereference -c %s -- '{path}'"
retval = exec_remote_command(auth_header, system_name, system_addr, action)
if retval["error"] != 0:
error_str = retval["msg"]
error_code = retval["error"]
service_msg = "Failed to view file content"
ret_data = check_command_error(error_str, error_code, service_msg)
# if generic "error" not in the dict
try:
jsonify(description=ret_data["description"],
error=ret_data["error"]
), ret_data["status_code"], ret_data["header"]
except:
return jsonify(description=ret_data["description"]
), ret_data["status_code"], ret_data["header"]
file_size = int(retval["msg"]) # in bytes
max_file_size = MAX_FILE_SIZE * (1024 * 1024)
if file_size > max_file_size:
app.logger.warning("File size exceeds limit")
# custom error raises when file size > SIZE_LIMIT env var
header = {"X-Size-Limit": "File exceeds size limit"}
return jsonify(description="Failed to view file content"), 400, header
# download with base64 to avoid encoding conversion and string processing
action = f"timeout {UTILITIES_TIMEOUT} head -c {max_file_size} -- '{path}'"
retval = exec_remote_command(auth_header, system_name, system_addr, action)
if retval["error"] != 0:
error_str = retval["msg"]
error_code = retval["error"]
service_msg = "Failed to view file content"
ret_data = check_command_error(error_str, error_code, service_msg)
# if generic "error" not in the dict
try:
return jsonify(description=ret_data["description"],
error=ret_data["error"]
), ret_data["status_code"], ret_data["header"]
except:
return jsonify(description=ret_data["description"]
), ret_data["status_code"], ret_data["header"]
content = retval["msg"].replace("$", "\n")
return jsonify(description="File content successfully returned",
output=content), 200
def list_directory():
auth_header = request.headers[AUTH_HEADER_NAME]
try:
system_name = request.headers["X-Machine-Name"]
except KeyError as e:
app.logger.error("No machinename given")
return jsonify(description="No machine name given"), 400
# PUBLIC endpoints from Kong to users
if system_name not in SYSTEMS_PUBLIC:
header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
return jsonify(description="Error listing contents of path",
error="Machine does not exist"), 400, header
# select index in the list corresponding with machine name
system_idx = SYSTEMS_PUBLIC.index(system_name)
system_addr = SYS_INTERNALS[system_idx]
try:
path = request.args.get("targetPath")
except BadRequestKeyError:
return jsonify(description="Error in ls operation",
error="'targetPath' query string missing"), 400
if path == None:
return jsonify(description="Error listing contents of path",
error="path query string missing"), 400
# if set shows entrys starting with . (not including . and/or .. dirs)
try:
showhidden = request.args.get("showhidden", None)
except BadRequestKeyError:
return jsonify(description="Error in ls operation",
error="option error"), 400
showall = ""
if showhidden != None:
showall = "-A"
action = f"timeout {UTILITIES_TIMEOUT} ls -l {showall} --time-style=+%Y-%m-%dT%H:%M:%S -- '{path}'"
retval = exec_remote_command(auth_header, system_name, system_addr, action)
if retval["error"] != 0:
error_str = retval["msg"]
error_code = retval["error"]
service_msg = "Error listing contents of path"
ret_data = check_command_error(error_str, error_code, service_msg)
# if generic "error" not in the dict
try:
jsonify(description=ret_data["description"],
error=ret_data["error"]
), ret_data["status_code"], ret_data["header"]
except:
return jsonify(description=ret_data["description"]
), ret_data["status_code"], ret_data["header"]
# file List is retorned as a string separated for a $ character
fileList = []
if len(retval["msg"].split("$")) == 1:
# if only one line is returned, there are two cases:
# 1. 'total 0': means directory was empty, so fileList is kept empty
# 2. 'r..... some_file.txt': means 'ls' was to only one file: 'ls /home/user/some.txt'
if retval["msg"][0:5] != 'total':
fileList = retval["msg"].split("$")
else:
fileList = retval["msg"].split("$")[1:]
totalSize = len(fileList)
# if pageSize and number were set:
pageSize = request.args.get("pageSize")
pageNumber = request.args.get("pageNumber")
app.logger.info(f"PageSize: {pageSize}. PageNumber: {pageNumber}")
# calculate the list to retrieve
if pageSize and pageNumber:
pageNumber = float(pageNumber)
pageSize = float(pageSize)
totalPages = int(ceil(float(totalSize) / float(pageSize)))
app.logger.info(f"Total Size: {totalSize}")
app.logger.info(f"Total Pages: {totalPages}")
if pageNumber < 1 or pageNumber > totalPages:
app.logger.warning(
"pageNumber ({pageNumber}) greater than total pages ({totalPages})"
.format(pageNumber=pageNumber, totalPages=totalPages))
app.logger.warning("Showing all results")
else:
beg_reg = int((pageNumber - 1) * pageSize)
end_reg = int(pageNumber * pageSize - 1)
app.logger.info(
"Initial reg {beg_reg}, final reg: {end_reg}".format(
beg_reg=beg_reg, end_reg=end_reg))
fileList = fileList[beg_reg:end_reg + 1]
outLabels = [
"name", "type", "link_target", "user", "group", "permissions",
"last_modified", "size"
]
# labels taken from list to dict with default value: ""
outList = []
logging.info("Length of file list: {}".format(len(fileList)))
for files in fileList:
line = files.split()
try:
symlink = line[8] # because of the -> which is 7
except IndexError:
symlink = ""
outDict = {
outLabels[0]: line[6],
outLabels[1]: line[0][0],
outLabels[2]: symlink,
outLabels[3]: line[2],
outLabels[4]: line[3],
outLabels[5]: line[0][1:],
outLabels[6]: line[5],
outLabels[7]: line[4]
}
outList.append(outDict)
return jsonify(descr="List of contents of path", output=outList), 200
def chown():
auth_header = request.headers[AUTH_HEADER_NAME]
try:
system_name = request.headers["X-Machine-Name"]
except KeyError as e:
app.logger.error("No machinename given")
return jsonify(description="No machine name given"), 400
# PUBLIC endpoints from Kong to users
if system_name not in SYSTEMS_PUBLIC:
header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
return jsonify(description="Error in chown operation",
error="Machine does not exist"), 400, header
# select index in the list corresponding with machine name
system_idx = SYSTEMS_PUBLIC.index(system_name)
system_addr = SYS_INTERNALS[system_idx]
try:
path = request.form["targetPath"]
if path == "":
return jsonify(description="Error in chown operation",
error="'targetPath' value is empty"), 400
except BadRequestKeyError:
return jsonify(description="Error in chown operation",
error="'targetPath' query string missing"), 400
if path == None:
return jsonify(description="Error in chown operation",
error="'targetPath' query string missing"), 400
try:
owner = request.form["owner"]
except Exception:
owner = ""
try:
group = request.form["group"]
except Exception:
group = ""
if owner == "" and group == "":
return jsonify(description="Error in chown operation",
error="group and/or owner should be set"), 400
action = f"timeout {UTILITIES_TIMEOUT} chown -v '{owner}':'{group}' -- '{path}'"
retval = exec_remote_command(auth_header, system_name, system_addr, action)
if retval["error"] != 0:
error_str = retval["msg"]
error_code = retval["error"]
service_msg = "Error in chown operation"
ret_data = check_command_error(error_str, error_code, service_msg)
# if generic "error" not in the dict
try:
jsonify(description=ret_data["description"],
error=ret_data["error"]
), ret_data["status_code"], ret_data["header"]
except:
return jsonify(description=ret_data["description"]
), ret_data["status_code"], ret_data["header"]
return jsonify(description="Operation completed", out=retval["msg"]), 200
def chmod():
auth_header = request.headers[AUTH_HEADER_NAME]
try:
system_name = request.headers["X-Machine-Name"]
except KeyError as e:
app.logger.error("No machinename given")
return jsonify(description="No machine name given"), 400
# PUBLIC endpoints from Kong to users
if system_name not in SYSTEMS_PUBLIC:
header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
return jsonify(description="Error in chmod operation",
error="Machine does not exist"), 400, header
# select index in the list corresponding with machine name
system_idx = SYSTEMS_PUBLIC.index(system_name)
system_addr = SYS_INTERNALS[system_idx]
# getting path from request form
try:
path = request.form["targetPath"]
if path == "":
return jsonify(description="Error in chmod operation",
error="'targetPath' value is empty"), 400
except BadRequestKeyError:
return jsonify(description="Error in chmod operation",
error="'targetPath' query string missing"), 400
# getting chmode's mode from request form:
try:
mode = request.form["mode"]
if mode == "":
return jsonify(description="Error in chmod operation",
error="'mode' value is empty"), 400
except BadRequestKeyError:
return jsonify(description="Error in chmod operation",
error="mode query string missing"), 400
# using -c flag for verbose mode in stdout
action = f"timeout {UTILITIES_TIMEOUT} chmod -v '{mode}' -- '{path}'"
retval = exec_remote_command(auth_header, system_name, system_addr, action)
if retval["error"] != 0:
error_str = retval["msg"]
error_code = retval["error"]
service_msg = "Error in chmod operation"
ret_data = check_command_error(error_str, error_code, service_msg)
# if generic "error" not in the dict
try:
jsonify(description=ret_data["description"],
error=ret_data["error"]
), ret_data["status_code"], ret_data["header"]
except:
return jsonify(description=ret_data["description"]
), ret_data["status_code"], ret_data["header"]
return jsonify(description="Operation completed", out=retval["msg"]), 200
def internal_operation(request, command):
system_idx = SYSTEMS_PUBLIC.index(STORAGE_JOBS_MACHINE)
system_addr = SYS_INTERNALS_UTILITIES[system_idx]
system_name = STORAGE_JOBS_MACHINE
targetPath = request.form.get("targetPath",
None) # path to save file in cluster
v = validate_input(targetPath)
if v != "":
return jsonify(description=f"Error on {command} operation",
error=f"'targetPath' {v}"), 400
[headers, ID] = get_tracing_headers(request)
# using actual_command to add options to check sanity of the command to be executed
actual_command = ""
if command in ['cp', 'mv', 'rsync']:
sourcePath = request.form.get("sourcePath",
None) # path to get file in cluster
v = validate_input(sourcePath)
if v != "":
return jsonify(description=f"Error on {command} operation",
error=f"'sourcePath' {v}"), 400
# checks if file to copy, move or rsync (targetPath) is a valid path
# remove the last part of the path (after last "/" char) to check if the dir can be written by user
_targetPath = targetPath.split("/")[:-1]
_targetPath = "/".join(_targetPath)
app.logger.info(f"_targetPath={_targetPath}")
check_dir = is_valid_dir(_targetPath, headers, system_name,
system_addr)
if not check_dir["result"]:
return jsonify(
description="targetPath error"), 400, check_dir["headers"]
check_file = is_valid_file(sourcePath, headers, system_name,
system_addr)
if not check_file["result"]:
check_dir = is_valid_dir(sourcePath, headers, system_name,
system_addr)
if not check_dir["result"]:
return jsonify(
description="sourcePath error"), 400, check_dir["headers"]
if command == "cp":
actual_command = "cp --force -dR --preserve=all -- "
elif command == "mv":
actual_command = "mv --force -- "
else:
actual_command = "rsync -av -- "
elif command == "rm":
# for 'rm' there's no source, set empty to call exec_internal_command(...)
# checks if file or dir to delete (targetPath) is a valid path or valid directory
check_file = is_valid_file(targetPath, headers, system_name,
system_addr)
if not check_file["result"]:
check_dir = is_valid_dir(targetPath, headers, system_name,
system_addr)
if not check_dir["result"]:
return jsonify(
description="targetPath error"), 400, check_dir["headers"]
sourcePath = ""
actual_command = "rm -rf -- "
else:
return jsonify(error=f"Command {command} not allowed"), 400
# don't add tracing ID, we'll be executed by srun
actual_command = f"{actual_command} '{sourcePath}' '{targetPath}'"
jobName = request.form.get("jobName", "") # jobName for SLURM
if jobName == "":
jobName = command + "-job"
app.logger.info(f"jobName not found, setting default to: {jobName}")
else:
v = validate_input(jobName)
if v != "":
return jsonify(description="Invalid jobName",
error=f"'jobName' {v}"), 400
try:
jobTime = request.form["time"] # job time, default is 2:00:00 H:M:s
if not job_time.check_jobTime(jobTime):
return jsonify(error="Not supported time format"), 400
except:
jobTime = "02:00:00"
stageOutJobId = request.form.get(
"stageOutJobId", None) # start after this JobId has finished
if stageOutJobId != None:
v = validate_input(stageOutJobId)
if v != "":
return jsonify(description="Invalid stageOutJobId",
error=f"'stageOutJobId' {v}"), 400
# select index in the list corresponding with machine name
system_idx = SYSTEMS_PUBLIC.index(STORAGE_JOBS_MACHINE)
system_addr = SYS_INTERNALS[system_idx]
app.logger.info(f"USE_SLURM_ACCOUNT: {USE_SLURM_ACCOUNT}")
# get "account" parameter, if not found, it is obtained from "id" command
try:
account = request.form["account"]
v = validate_input(account)
if v != "":
return jsonify(description="Invalid account",
error=f"'account' {v}"), 400
except:
if USE_SLURM_ACCOUNT:
username = get_username(headers[AUTH_HEADER_NAME])
id_command = f"ID={ID} timeout {UTILITIES_TIMEOUT} id -gn -- {username}"
resp = exec_remote_command(headers, STORAGE_JOBS_MACHINE,
system_addr, id_command)
if resp["error"] != 0:
retval = check_command_error(resp["msg"], resp["error"],
f"{command} job")
return jsonify(description=f"Failed to submit {command} job",
error=retval["description"]
), retval["status_code"], retval["header"]
account = resp["msg"]
else:
account = None
# check if machine is accessible by user:
# exec test remote command
resp = exec_remote_command(headers, STORAGE_JOBS_MACHINE, system_addr,
f"ID={ID} true")
if resp["error"] != 0:
error_str = resp["msg"]
if resp["error"] == -2:
header = {"X-Machine-Not-Available": "Machine is not available"}
return jsonify(
description=f"Failed to submit {command} job"), 400, header
if in_str(error_str, "Permission") or in_str(error_str, "OPENSSH"):
header = {
"X-Permission-Denied":
"User does not have permissions to access machine or path"
}
return jsonify(
description=f"Failed to submit {command} job"), 404, header
retval = exec_internal_command(headers, actual_command, jobName, jobTime,
stageOutJobId, account)
# returns "error" key or "success" key
try:
error = retval["error"]
errmsg = retval["msg"]
desc = retval["desc"]
# headers values cannot contain "\n" strings
return jsonify(error=desc), 400, {"X-Sbatch-Error": errmsg}
except KeyError:
success = retval["success"]
task_id = retval["task_id"]
return jsonify(success=success, task_id=task_id), 201