Beispiel #1
0
def symlink():

    auth_header = request.headers[AUTH_HEADER_NAME]

    try:
        system_name = request.headers["X-Machine-Name"]
    except KeyError as e:
        app.logger.error("No machinename given")
        return jsonify(description="No machine name given"), 400

    # PUBLIC endpoints from Kong to users
    if system_name not in SYSTEMS_PUBLIC:
        header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
        return jsonify(description="Failed to create symlink",
                       error="Machine does not exist"), 400, header

    # select index in the list corresponding with machine name
    system_idx = SYSTEMS_PUBLIC.index(system_name)
    system_addr = SYS_INTERNALS[system_idx]

    try:
        linkPath = request.form["linkPath"]
        if linkPath == "":
            return jsonify(description="Failed to create symlink",
                           error="'linkPath' value is empty"), 400
    except BadRequestKeyError:
        return jsonify(description="Failed to create symlink",
                       error="'linkPath' query string missing"), 400

    try:
        targetPath = request.form["targetPath"]
        if targetPath == "":
            return jsonify(description="Failed to create symlink",
                           error="'targetPath' value is empty"), 400
    except BadRequestKeyError:
        return jsonify(description="Failed to create symlink",
                       error="'targetPath' query string missing"), 400

    action = f"timeout {UTILITIES_TIMEOUT} ln -s -- '{targetPath}' '{linkPath}'"

    retval = exec_remote_command(auth_header, system_name, system_addr, action)

    if retval["error"] != 0:
        error_str = retval["msg"]
        error_code = retval["error"]
        service_msg = "Failed to create symlink"

        ret_data = check_command_error(error_str, error_code, service_msg)

        # if generic "error" not in the dict
        try:
            jsonify(description=ret_data["description"],
                    error=ret_data["error"]
                    ), ret_data["status_code"], ret_data["header"]
        except:
            return jsonify(description=ret_data["description"]
                           ), ret_data["status_code"], ret_data["header"]

    return jsonify(description="Success create the symlink"), 201
Beispiel #2
0
def checksum():

    auth_header = request.headers[AUTH_HEADER_NAME]

    try:
        system_name = request.headers["X-Machine-Name"]
    except KeyError as e:
        app.logger.error("No machinename given")
        return jsonify(description="No machine name given"), 400

    # PUBLIC endpoints from Kong to users
    if system_name not in SYSTEMS_PUBLIC:
        header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
        return jsonify(description="Error obatining checksum",
                       error="Machine does not exist"), 400, header

    # select index in the list corresponding with machine name
    system_idx = SYSTEMS_PUBLIC.index(system_name)
    system_addr = SYS_INTERNALS[system_idx]

    try:
        path = request.args.get("targetPath")
        if path == "":
            return jsonify(description="Error obatining checksum",
                           error="'targetPath' value is empty"), 400

    except BadRequestKeyError:
        return jsonify(description="Error obatining checksum",
                       error="'targetPath' query string missing"), 400

    action = f"timeout {UTILITIES_TIMEOUT} sha256sum -- '{path}'"

    retval = exec_remote_command(auth_header, system_name, system_addr, action)

    if retval["error"] != 0:

        error_str = retval["msg"]
        error_code = retval["error"]
        service_msg = "Error obtaining checksum"

        ret_data = check_command_error(error_str, error_code, service_msg)

        # if generic "error" not in the dict
        try:
            jsonify(description=ret_data["description"],
                    error=ret_data["error"]
                    ), ret_data["status_code"], ret_data["header"]
        except:
            return jsonify(description=ret_data["description"]
                           ), ret_data["status_code"], ret_data["header"]

    # on success: retval["msg"] = "checksum  /path/to/file"
    output = retval["msg"].split()[0]

    return jsonify(description="Checksum successfully retrieved",
                   output=output), 200
Beispiel #3
0
def rm():

    auth_header = request.headers[AUTH_HEADER_NAME]

    try:
        system_name = request.headers["X-Machine-Name"]
    except KeyError as e:
        app.logger.error("No machinename given")
        return jsonify(description="No machine name given"), 400

    # PUBLIC endpoints from Kong to users
    if system_name not in SYSTEMS_PUBLIC:
        header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
        return jsonify(description="Error on delete operation",
                       error="Machine does not exist"), 400, header

    # select index in the list corresponding with machine name
    system_idx = SYSTEMS_PUBLIC.index(system_name)
    system_addr = SYS_INTERNALS[system_idx]

    try:
        path = request.form["targetPath"]
        if path == "":
            return jsonify(description="Error on delete operation",
                           error="'targetPath' value is empty"), 400
    except BadRequestKeyError:
        return jsonify(description="Error on delete operation",
                       error="'targetPath' query string missing"), 400

    # action to execute
    # -r is for recursivelly delete files into directories
    action = f"timeout {UTILITIES_TIMEOUT} rm -r --interactive=never -- '{path}'"

    retval = exec_remote_command(auth_header, system_name, system_addr, action)

    if retval["error"] != 0:
        error_str = retval["msg"]
        error_code = retval["error"]
        service_msg = "Error on delete operation"

        ret_data = check_command_error(error_str, error_code, service_msg)

        # if generic "error" not in the dict
        try:
            jsonify(description=ret_data["description"],
                    error=ret_data["error"]
                    ), ret_data["status_code"], ret_data["header"]
        except:
            return jsonify(description=ret_data["description"]
                           ), ret_data["status_code"], ret_data["header"]

    return jsonify(description="Success to delete file or directory.",
                   output=""), 204
Beispiel #4
0
def common_fs_operation(request, command):
    try:
        system_name = request.headers["X-Machine-Name"]
    except KeyError as e:
        app.logger.error("No machinename given")
        return jsonify(description="No machine name given"), 400

    # PUBLIC endpoints from Kong to users
    if system_name not in SYSTEMS_PUBLIC:
        header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
        return jsonify(description=f"Error on {command} operation",
                       error="Machine does not exist"), 400, header

    # select index in the list corresponding with machine name
    system_idx = SYSTEMS_PUBLIC.index(system_name)
    system_addr = SYS_INTERNALS[system_idx]

    # get targetPath to apply command
    tn = 'targetPath'
    if request.method == 'GET':
        targetPath = request.args.get("targetPath", None)
        if (targetPath == None) and (command in ['base64', 'stat']):
            # TODO: review API
            tn = "sourcePath"
            targetPath = request.args.get("sourcePath", None)
    else:  # DELETE, POST, PUT
        targetPath = request.form.get("targetPath", None)

    v = validate_input(targetPath)
    if v != "":
        return jsonify(description=f"Error on {command} operation",
                       error=f"'{tn}' {v}"), 400

    if command in ['copy', 'rename']:
        sourcePath = request.form.get("sourcePath", None)
        v = validate_input(sourcePath)
        if v != "":
            return jsonify(description=f"Error on {command} operation",
                           error=f"'sourcePath' {v}"), 400

    file_content = None
    file_transfer = None
    success_code = 200

    if command == "base64":
        action = f"base64 --wrap=0 -- '{targetPath}'"
        file_transfer = 'download'
    elif command == "checksum":
        action = f"sha256sum -- '{targetPath}'"
    elif command == "chmod":
        mode = request.form.get("mode", None)
        v = validate_input(mode)
        if v != "":
            return jsonify(description="Error on chmod operation",
                           error=f"'mode' {v}"), 400
        action = f"chmod -v '{mode}' -- '{targetPath}'"
    elif command == "chown":
        owner = request.form.get("owner", "")
        group = request.form.get("group", "")
        if owner == "" and group == "":
            return jsonify(description="Error in chown operation",
                           error="group or owner must be set"), 400
        v = validate_input(owner + group)
        if v != "":
            return jsonify(description="Error in chown operation",
                           error=f"group or owner {v}"), 400
        action = f"chown -v '{owner}':'{group}' -- '{targetPath}'"
    elif command == "copy":
        # -r is for recursivelly copy files into directories
        action = f"cp --force -dR --preserve=all -- '{sourcePath}' '{targetPath}'"
        success_code = 201
    elif command == "file":
        # -b: do not prepend filenames to output lines
        action = f"file -b -- '{targetPath}'"
    elif command == "head":
        action = f"head -c {MAX_FILE_SIZE_BYTES} -- '{targetPath}'"
        file_transfer = 'download'
    elif command == "ls":
        # if set shows entrys starting with . (not including . and/or .. dirs)
        showhidden = request.args.get("showhidden", None)
        showall = ""
        if showhidden != None:
            showall = "-A"
        action = f"ls -l {showall} --time-style=+%Y-%m-%dT%H:%M:%S -- '{targetPath}'"
    elif command == "mkdir":
        try:
            p = request.form["p"]
            parent = "-p"
        except BadRequestKeyError:
            parent = ""
        action = f"mkdir {parent} -- '{targetPath}'"
        success_code = 201
    elif command == "rename":
        action = f"mv --force -- '{sourcePath}' '{targetPath}'"
    elif command == "rm":
        # -r is for recursivelly delete files into directories
        action = f"rm -r --interactive=never -- '{targetPath}'"
        success_code = 204
    elif command == "stat":
        action = f"stat --dereference -c %s -- '{targetPath}'"
    elif command == "symlink":
        linkPath = request.form.get("linkPath", None)
        v = validate_input(linkPath)
        if v != "":
            return jsonify(description="Failed to create symlink",
                           error=f"'linkPath' value {v}"), 400
        action = f"ln -s -- '{targetPath}' '{linkPath}'"
        success_code = 201
    elif command == "upload":
        try:
            if 'file' not in request.files:
                return jsonify(description="Failed to upload file",
                               error="No file in query"), 400
            file = request.files['file']
            app.logger.info(f"Upload length: {file.content_length}")
            v = validate_input(file.filename)
            if v != "":
                return jsonify(description="Failed to upload file",
                               error=f"Filename {v}"), 400
        except:
            return jsonify(description='Error on upload operation',
                           output=''), 400
        filename = secure_filename(file.filename)
        action = f"cat > '{targetPath}/{filename}'"
        file_content = file.read()
        file_transfer = 'upload'
        success_code = 201
    else:
        app.logger.error(f"Unknown command on common_fs_operation: {command}")
        return jsonify(description="Error on internal operation",
                       error="Internal error"), 400

    [headers, ID] = get_tracing_headers(request)
    action = f"ID={ID} timeout {UTILITIES_TIMEOUT} {action}"
    retval = exec_remote_command(headers, system_name, system_addr, action,
                                 file_transfer, file_content)

    if retval["error"] != 0:
        error_str = retval["msg"]
        error_code = retval["error"]
        service_msg = f"Error on {command} operation"

        ret_data = check_command_error(error_str, error_code, service_msg)

        # if generic "error" not in the dict
        try:
            return jsonify(description=ret_data["description"],
                           error=ret_data["error"]
                           ), ret_data["status_code"], ret_data["header"]
        except:
            return jsonify(description=ret_data["description"]
                           ), ret_data["status_code"], ret_data["header"]

    description = f"Success to {command} file or directory."
    output = ''
    if command == 'checksum':
        # return only hash, msg sintax:  hash filename
        output = retval["msg"].split()[0]
    elif command in ['base64', 'chmod', 'chown', 'file', 'head', 'stat']:
        output = retval["msg"]
    elif command == 'ls':
        description = "List of contents"
        output = ls_parse(request, retval)
    elif command == "upload":
        description = "File upload successful"

    return jsonify(description=description, output=output), success_code
Beispiel #5
0
def common_operation(request, command, method):

    auth_header = request.headers[AUTH_HEADER_NAME]

    try:
        system_name = request.headers["X-Machine-Name"]
    except KeyError as e:
        app.logger.error("No machinename given")
        return jsonify(description="No machine name given"), 400

    # PUBLIC endpoints from Kong to users
    if system_name not in SYSTEMS_PUBLIC:
        header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
        return jsonify(description="Error on " + command + " operation",
                       error="Machine does not exist"), 400, header

    # select index in the list corresponding with machine name
    system_idx = SYSTEMS_PUBLIC.index(system_name)
    system_addr = SYS_INTERNALS[system_idx]

    try:
        sourcePath = request.form["sourcePath"]
        if sourcePath == "":
            return jsonify(description="Error on " + command + " operation",
                           error="'sourcePath' value is empty"), 400
    except BadRequestKeyError:
        return jsonify(description="Error on " + command + " operation",
                       error="'sourcePath' query string missing"), 400

    try:
        targetPath = request.form["targetPath"]
        if targetPath == "":
            return jsonify(description="Error on " + command + " operation",
                           error="'targetPath' value is empty"), 400
    except BadRequestKeyError:
        return jsonify(description="Error on " + command + " operation",
                       error="target query string missing"), 400

    if command == "copy":
        # action to execute
        # -r is for recursivelly copy files into directories
        action = f"timeout {UTILITIES_TIMEOUT} cp --force -dR --preserve=all -- '{sourcePath}' '{targetPath}'"
        success_code = 201
    elif command == "rename":
        action = f"timeout {UTILITIES_TIMEOUT} mv --force -- '{sourcePath}' '{targetPath}'"
        success_code = 200
    else:
        app.logger.error("Unknown command on common_operation: " + command)
        return jsonify(description="Error on unkownon operation",
                       error="Unknown"), 400

    retval = exec_remote_command(auth_header, system_name, system_addr, action)

    if retval["error"] != 0:
        error_str = retval["msg"]
        error_code = retval["error"]
        service_msg = f"Error on {command} operation"

        ret_data = check_command_error(error_str, error_code, service_msg)

        # if generic "error" not in the dict
        try:
            jsonify(description=ret_data["description"],
                    error=ret_data["error"]
                    ), ret_data["status_code"], ret_data["header"]
        except:
            return jsonify(description=ret_data["description"]
                           ), ret_data["status_code"], ret_data["header"]

    return jsonify(description="Success to " + command + " file or directory.",
                   output=""), success_code
Beispiel #6
0
def view():

    auth_header = request.headers[AUTH_HEADER_NAME]

    try:
        system_name = request.headers["X-Machine-Name"]
    except KeyError as e:
        app.logger.error("No machinename given")
        return jsonify(description="No machine name given"), 400

    # PUBLIC endpoints from Kong to users
    if system_name not in SYSTEMS_PUBLIC:
        header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
        return jsonify(description="Failed to view file content",
                       error="Machine does not exist"), 400, header

    # select index in the list corresponding with machine name
    system_idx = SYSTEMS_PUBLIC.index(system_name)
    system_addr = SYS_INTERNALS[system_idx]

    path = request.args.get("targetPath")

    if path == None:
        return jsonify(description="Failed to view file content",
                       error="'targetPath' query string missing"), 400
    if path == "":
        return jsonify(description="Failed to view file content",
                       error="'targetPath' value is empty"), 400

    # check file size
    action = f"timeout {UTILITIES_TIMEOUT} stat --dereference -c %s -- '{path}'"
    retval = exec_remote_command(auth_header, system_name, system_addr, action)

    if retval["error"] != 0:

        error_str = retval["msg"]
        error_code = retval["error"]
        service_msg = "Failed to view file content"

        ret_data = check_command_error(error_str, error_code, service_msg)

        # if generic "error" not in the dict
        try:
            jsonify(description=ret_data["description"],
                    error=ret_data["error"]
                    ), ret_data["status_code"], ret_data["header"]
        except:
            return jsonify(description=ret_data["description"]
                           ), ret_data["status_code"], ret_data["header"]

    file_size = int(retval["msg"])  # in bytes
    max_file_size = MAX_FILE_SIZE * (1024 * 1024)

    if file_size > max_file_size:
        app.logger.warning("File size exceeds limit")
        # custom error raises when file size > SIZE_LIMIT env var
        header = {"X-Size-Limit": "File exceeds size limit"}
        return jsonify(description="Failed to view file content"), 400, header

    # download with base64 to avoid encoding conversion and string processing
    action = f"timeout {UTILITIES_TIMEOUT} head -c {max_file_size} -- '{path}'"
    retval = exec_remote_command(auth_header, system_name, system_addr, action)
    if retval["error"] != 0:

        error_str = retval["msg"]
        error_code = retval["error"]
        service_msg = "Failed to view file content"

        ret_data = check_command_error(error_str, error_code, service_msg)

        # if generic "error" not in the dict
        try:
            return jsonify(description=ret_data["description"],
                           error=ret_data["error"]
                           ), ret_data["status_code"], ret_data["header"]
        except:
            return jsonify(description=ret_data["description"]
                           ), ret_data["status_code"], ret_data["header"]

    content = retval["msg"].replace("$", "\n")

    return jsonify(description="File content successfully returned",
                   output=content), 200
Beispiel #7
0
def list_directory():

    auth_header = request.headers[AUTH_HEADER_NAME]

    try:
        system_name = request.headers["X-Machine-Name"]
    except KeyError as e:
        app.logger.error("No machinename given")
        return jsonify(description="No machine name given"), 400

    # PUBLIC endpoints from Kong to users
    if system_name not in SYSTEMS_PUBLIC:
        header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
        return jsonify(description="Error listing contents of path",
                       error="Machine does not exist"), 400, header

    # select index in the list corresponding with machine name
    system_idx = SYSTEMS_PUBLIC.index(system_name)
    system_addr = SYS_INTERNALS[system_idx]

    try:
        path = request.args.get("targetPath")
    except BadRequestKeyError:
        return jsonify(description="Error in ls operation",
                       error="'targetPath' query string missing"), 400

    if path == None:
        return jsonify(description="Error listing contents of path",
                       error="path query string missing"), 400

    # if set shows entrys starting with . (not including . and/or .. dirs)
    try:
        showhidden = request.args.get("showhidden", None)
    except BadRequestKeyError:
        return jsonify(description="Error in ls operation",
                       error="option error"), 400

    showall = ""
    if showhidden != None:
        showall = "-A"

    action = f"timeout {UTILITIES_TIMEOUT} ls -l {showall} --time-style=+%Y-%m-%dT%H:%M:%S -- '{path}'"

    retval = exec_remote_command(auth_header, system_name, system_addr, action)

    if retval["error"] != 0:
        error_str = retval["msg"]
        error_code = retval["error"]
        service_msg = "Error listing contents of path"

        ret_data = check_command_error(error_str, error_code, service_msg)

        # if generic "error" not in the dict
        try:
            jsonify(description=ret_data["description"],
                    error=ret_data["error"]
                    ), ret_data["status_code"], ret_data["header"]
        except:
            return jsonify(description=ret_data["description"]
                           ), ret_data["status_code"], ret_data["header"]

    # file List is retorned as a string separated for a $ character
    fileList = []
    if len(retval["msg"].split("$")) == 1:
        # if only one line is returned, there are two cases:
        # 1. 'total 0': means directory was empty, so fileList is kept empty
        # 2. 'r.....   some_file.txt': means 'ls' was to only one file: 'ls /home/user/some.txt'
        if retval["msg"][0:5] != 'total':
            fileList = retval["msg"].split("$")
    else:
        fileList = retval["msg"].split("$")[1:]

    totalSize = len(fileList)

    # if pageSize and number were set:
    pageSize = request.args.get("pageSize")
    pageNumber = request.args.get("pageNumber")

    app.logger.info(f"PageSize: {pageSize}. PageNumber: {pageNumber}")

    # calculate the list to retrieve
    if pageSize and pageNumber:

        pageNumber = float(pageNumber)
        pageSize = float(pageSize)

        totalPages = int(ceil(float(totalSize) / float(pageSize)))

        app.logger.info(f"Total Size: {totalSize}")
        app.logger.info(f"Total Pages: {totalPages}")

        if pageNumber < 1 or pageNumber > totalPages:
            app.logger.warning(
                "pageNumber ({pageNumber}) greater than total pages ({totalPages})"
                .format(pageNumber=pageNumber, totalPages=totalPages))
            app.logger.warning("Showing all results")
        else:
            beg_reg = int((pageNumber - 1) * pageSize)
            end_reg = int(pageNumber * pageSize - 1)

            app.logger.info(
                "Initial reg {beg_reg}, final reg: {end_reg}".format(
                    beg_reg=beg_reg, end_reg=end_reg))

            fileList = fileList[beg_reg:end_reg + 1]

    outLabels = [
        "name", "type", "link_target", "user", "group", "permissions",
        "last_modified", "size"
    ]

    # labels taken from list to dict with default value: ""
    outList = []

    logging.info("Length of file list: {}".format(len(fileList)))

    for files in fileList:

        line = files.split()

        try:
            symlink = line[8]  # because of the -> which is 7
        except IndexError:
            symlink = ""

        outDict = {
            outLabels[0]: line[6],
            outLabels[1]: line[0][0],
            outLabels[2]: symlink,
            outLabels[3]: line[2],
            outLabels[4]: line[3],
            outLabels[5]: line[0][1:],
            outLabels[6]: line[5],
            outLabels[7]: line[4]
        }

        outList.append(outDict)

    return jsonify(descr="List of contents of path", output=outList), 200
Beispiel #8
0
def chown():

    auth_header = request.headers[AUTH_HEADER_NAME]

    try:
        system_name = request.headers["X-Machine-Name"]
    except KeyError as e:
        app.logger.error("No machinename given")
        return jsonify(description="No machine name given"), 400

    # PUBLIC endpoints from Kong to users
    if system_name not in SYSTEMS_PUBLIC:
        header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
        return jsonify(description="Error in chown operation",
                       error="Machine does not exist"), 400, header

    # select index in the list corresponding with machine name
    system_idx = SYSTEMS_PUBLIC.index(system_name)
    system_addr = SYS_INTERNALS[system_idx]

    try:
        path = request.form["targetPath"]
        if path == "":
            return jsonify(description="Error in chown operation",
                           error="'targetPath' value is empty"), 400
    except BadRequestKeyError:
        return jsonify(description="Error in chown operation",
                       error="'targetPath' query string missing"), 400

    if path == None:
        return jsonify(description="Error in chown operation",
                       error="'targetPath' query string missing"), 400
    try:
        owner = request.form["owner"]
    except Exception:
        owner = ""

    try:
        group = request.form["group"]
    except Exception:
        group = ""

    if owner == "" and group == "":
        return jsonify(description="Error in chown operation",
                       error="group and/or owner should be set"), 400

    action = f"timeout {UTILITIES_TIMEOUT} chown -v '{owner}':'{group}' -- '{path}'"

    retval = exec_remote_command(auth_header, system_name, system_addr, action)

    if retval["error"] != 0:
        error_str = retval["msg"]
        error_code = retval["error"]
        service_msg = "Error in chown operation"

        ret_data = check_command_error(error_str, error_code, service_msg)

        # if generic "error" not in the dict
        try:
            jsonify(description=ret_data["description"],
                    error=ret_data["error"]
                    ), ret_data["status_code"], ret_data["header"]
        except:
            return jsonify(description=ret_data["description"]
                           ), ret_data["status_code"], ret_data["header"]

    return jsonify(description="Operation completed", out=retval["msg"]), 200
Beispiel #9
0
def chmod():

    auth_header = request.headers[AUTH_HEADER_NAME]

    try:
        system_name = request.headers["X-Machine-Name"]
    except KeyError as e:
        app.logger.error("No machinename given")
        return jsonify(description="No machine name given"), 400

    # PUBLIC endpoints from Kong to users
    if system_name not in SYSTEMS_PUBLIC:
        header = {"X-Machine-Does-Not-Exist": "Machine does not exist"}
        return jsonify(description="Error in chmod operation",
                       error="Machine does not exist"), 400, header

    # select index in the list corresponding with machine name
    system_idx = SYSTEMS_PUBLIC.index(system_name)
    system_addr = SYS_INTERNALS[system_idx]

    # getting path from request form
    try:
        path = request.form["targetPath"]
        if path == "":
            return jsonify(description="Error in chmod operation",
                           error="'targetPath' value is empty"), 400
    except BadRequestKeyError:
        return jsonify(description="Error in chmod operation",
                       error="'targetPath' query string missing"), 400

    # getting chmode's mode from request form:
    try:
        mode = request.form["mode"]
        if mode == "":
            return jsonify(description="Error in chmod operation",
                           error="'mode' value is empty"), 400
    except BadRequestKeyError:
        return jsonify(description="Error in chmod operation",
                       error="mode query string missing"), 400

    # using -c flag for verbose mode in stdout
    action = f"timeout {UTILITIES_TIMEOUT} chmod -v '{mode}' -- '{path}'"

    retval = exec_remote_command(auth_header, system_name, system_addr, action)

    if retval["error"] != 0:
        error_str = retval["msg"]
        error_code = retval["error"]
        service_msg = "Error in chmod operation"

        ret_data = check_command_error(error_str, error_code, service_msg)

        # if generic "error" not in the dict
        try:
            jsonify(description=ret_data["description"],
                    error=ret_data["error"]
                    ), ret_data["status_code"], ret_data["header"]
        except:
            return jsonify(description=ret_data["description"]
                           ), ret_data["status_code"], ret_data["header"]

    return jsonify(description="Operation completed", out=retval["msg"]), 200
Beispiel #10
0
def internal_operation(request, command):

    system_idx = SYSTEMS_PUBLIC.index(STORAGE_JOBS_MACHINE)
    system_addr = SYS_INTERNALS_UTILITIES[system_idx]
    system_name = STORAGE_JOBS_MACHINE

    targetPath = request.form.get("targetPath",
                                  None)  # path to save file in cluster
    v = validate_input(targetPath)
    if v != "":
        return jsonify(description=f"Error on {command} operation",
                       error=f"'targetPath' {v}"), 400

    [headers, ID] = get_tracing_headers(request)
    # using actual_command to add options to check sanity of the command to be executed
    actual_command = ""
    if command in ['cp', 'mv', 'rsync']:
        sourcePath = request.form.get("sourcePath",
                                      None)  # path to get file in cluster
        v = validate_input(sourcePath)
        if v != "":
            return jsonify(description=f"Error on {command} operation",
                           error=f"'sourcePath' {v}"), 400

        # checks if file to copy, move or rsync (targetPath) is a valid path
        # remove the last part of the path (after last "/" char) to check if the dir can be written by user

        _targetPath = targetPath.split("/")[:-1]
        _targetPath = "/".join(_targetPath)

        app.logger.info(f"_targetPath={_targetPath}")

        check_dir = is_valid_dir(_targetPath, headers, system_name,
                                 system_addr)

        if not check_dir["result"]:
            return jsonify(
                description="targetPath error"), 400, check_dir["headers"]

        check_file = is_valid_file(sourcePath, headers, system_name,
                                   system_addr)

        if not check_file["result"]:
            check_dir = is_valid_dir(sourcePath, headers, system_name,
                                     system_addr)

            if not check_dir["result"]:
                return jsonify(
                    description="sourcePath error"), 400, check_dir["headers"]

        if command == "cp":
            actual_command = "cp --force -dR --preserve=all -- "
        elif command == "mv":
            actual_command = "mv --force -- "
        else:
            actual_command = "rsync -av -- "
    elif command == "rm":
        # for 'rm' there's no source, set empty to call exec_internal_command(...)
        # checks if file or dir to delete (targetPath) is a valid path or valid directory
        check_file = is_valid_file(targetPath, headers, system_name,
                                   system_addr)

        if not check_file["result"]:
            check_dir = is_valid_dir(targetPath, headers, system_name,
                                     system_addr)

            if not check_dir["result"]:
                return jsonify(
                    description="targetPath error"), 400, check_dir["headers"]

        sourcePath = ""
        actual_command = "rm -rf -- "
    else:
        return jsonify(error=f"Command {command} not allowed"), 400

    # don't add tracing ID, we'll be executed by srun
    actual_command = f"{actual_command} '{sourcePath}' '{targetPath}'"

    jobName = request.form.get("jobName", "")  # jobName for SLURM
    if jobName == "":
        jobName = command + "-job"
        app.logger.info(f"jobName not found, setting default to: {jobName}")
    else:
        v = validate_input(jobName)
        if v != "":
            return jsonify(description="Invalid jobName",
                           error=f"'jobName' {v}"), 400

    try:
        jobTime = request.form["time"]  # job time, default is 2:00:00 H:M:s
        if not job_time.check_jobTime(jobTime):
            return jsonify(error="Not supported time format"), 400
    except:
        jobTime = "02:00:00"

    stageOutJobId = request.form.get(
        "stageOutJobId", None)  # start after this JobId has finished
    if stageOutJobId != None:
        v = validate_input(stageOutJobId)
        if v != "":
            return jsonify(description="Invalid stageOutJobId",
                           error=f"'stageOutJobId' {v}"), 400

    # select index in the list corresponding with machine name
    system_idx = SYSTEMS_PUBLIC.index(STORAGE_JOBS_MACHINE)
    system_addr = SYS_INTERNALS[system_idx]

    app.logger.info(f"USE_SLURM_ACCOUNT: {USE_SLURM_ACCOUNT}")
    # get "account" parameter, if not found, it is obtained from "id" command
    try:
        account = request.form["account"]
        v = validate_input(account)
        if v != "":
            return jsonify(description="Invalid account",
                           error=f"'account' {v}"), 400
    except:
        if USE_SLURM_ACCOUNT:
            username = get_username(headers[AUTH_HEADER_NAME])
            id_command = f"ID={ID} timeout {UTILITIES_TIMEOUT} id -gn -- {username}"
            resp = exec_remote_command(headers, STORAGE_JOBS_MACHINE,
                                       system_addr, id_command)
            if resp["error"] != 0:
                retval = check_command_error(resp["msg"], resp["error"],
                                             f"{command} job")
                return jsonify(description=f"Failed to submit {command} job",
                               error=retval["description"]
                               ), retval["status_code"], retval["header"]

            account = resp["msg"]
        else:
            account = None

    # check if machine is accessible by user:
    # exec test remote command
    resp = exec_remote_command(headers, STORAGE_JOBS_MACHINE, system_addr,
                               f"ID={ID} true")

    if resp["error"] != 0:
        error_str = resp["msg"]
        if resp["error"] == -2:
            header = {"X-Machine-Not-Available": "Machine is not available"}
            return jsonify(
                description=f"Failed to submit {command} job"), 400, header
        if in_str(error_str, "Permission") or in_str(error_str, "OPENSSH"):
            header = {
                "X-Permission-Denied":
                "User does not have permissions to access machine or path"
            }
            return jsonify(
                description=f"Failed to submit {command} job"), 404, header

    retval = exec_internal_command(headers, actual_command, jobName, jobTime,
                                   stageOutJobId, account)

    # returns "error" key or "success" key
    try:
        error = retval["error"]
        errmsg = retval["msg"]
        desc = retval["desc"]
        # headers values cannot contain "\n" strings
        return jsonify(error=desc), 400, {"X-Sbatch-Error": errmsg}
    except KeyError:
        success = retval["success"]
        task_id = retval["task_id"]
        return jsonify(success=success, task_id=task_id), 201