示例#1
0
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id)
    client_dir = client_id_dir(client_id)
    status = returnvalues.OK
    defaults = signature()[1]
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
    )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    flags = ''.join(accepted['flags'])
    lines = int(accepted['lines'][-1])
    pattern_list = accepted['path']

    # Please note that base_dir must end in slash to avoid access to other
    # user dirs when own name is a prefix of another user name

    base_dir = os.path.abspath(
        os.path.join(configuration.user_home, client_dir)) + os.sep

    if verbose(flags):
        for flag in flags:
            output_objects.append({
                'object_type': 'text',
                'text': '%s using flag: %s' % (op_name, flag)
            })

    for pattern in pattern_list:

        # Check directory traversal attempts before actual handling to avoid
        # leaking information about file system layout while allowing
        # consistent error messages

        unfiltered_match = glob.glob(base_dir + pattern)
        match = []
        for server_path in unfiltered_match:
            # IMPORTANT: path must be expanded to abs for proper chrooting
            abs_path = os.path.abspath(server_path)
            if not valid_user_path(configuration, abs_path, base_dir, True):

                # out of bounds - save user warning for later to allow
                # partial match:
                # ../*/* is technically allowed to match own files.

                logger.warning('%s tried to %s restricted path %s ! (%s)' %
                               (client_id, op_name, abs_path, pattern))
                continue
            match.append(abs_path)

        # Now actually treat list of allowed matchings and notify if no
        # (allowed) match

        if not match:
            output_objects.append({
                'object_type': 'file_not_found',
                'name': pattern
            })
            status = returnvalues.FILE_NOT_FOUND

        for abs_path in match:
            relative_path = abs_path.replace(base_dir, '')
            output_lines = []

            # We search for the last 'lines' lines by beginning from the end of
            # the file and exponetially backtracking until the backtrack
            # contains at least 'lines' lines or we're back to the beginning of
            # the file.
            # At that point we skip any extra lines before printing.

            try:
                filedes = open(abs_path, 'r')

                # Go to end of file and backtrack

                backstep = 1
                newlines = 0
                filedes.seek(0, 2)
                length = filedes.tell()
                while backstep < length and newlines <= lines:
                    backstep *= 2
                    if backstep > length:
                        backstep = length
                    filedes.seek(-backstep, 2)
                    newlines = len(filedes.readlines())
                if length:

                    # Go back after reading to end of file

                    filedes.seek(-backstep, 2)

                # Skip any extra lines caused by the exponential backtracking.
                # We could probably speed up convergence with binary search...

                while newlines > lines:
                    dummy = filedes.readline()
                    newlines -= 1
                backstep = length - filedes.tell()

                # Now we're at the wanted spot - print rest of file

                for _ in range(newlines):
                    output_lines.append(filedes.readline())
                filedes.close()
            except Exception, exc:
                output_objects.append({
                    'object_type':
                    'error_text',
                    'text':
                    "%s: '%s': %s" % (op_name, relative_path, exc)
                })
                logger.error("%s: failed on '%s': %s" %
                             (op_name, relative_path, exc))
                status = returnvalues.SYSTEM_ERROR
                continue
            entry = {
                'object_type': 'file_output',
                'lines': output_lines,
                'wrap_binary': binary(flags),
                'wrap_targets': ['lines']
            }
            if verbose(flags):
                entry['path'] = relative_path
            output_objects.append(entry)
示例#2
0
文件: head.py 项目: heromod/migrid
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id)
    client_dir = client_id_dir(client_id)
    defaults = signature()[1]
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
        )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    flags = ''.join(accepted['flags'])
    lines = int(accepted['lines'][-1])
    pattern_list = accepted['path']

    # Please note that base_dir must end in slash to avoid access to other
    # user dirs when own name is a prefix of another user name

    base_dir = os.path.abspath(os.path.join(configuration.user_home,
                               client_dir)) + os.sep

    status = returnvalues.OK

    if verbose(flags):
        for flag in flags:
            output_objects.append({'object_type': 'text', 'text'
                                  : '%s using flag: %s' % (op_name,
                                  flag)})

    for pattern in pattern_list:

        # Check directory traversal attempts before actual handling to avoid
        # leaking information about file system layout while allowing
        # consistent error messages

        unfiltered_match = glob.glob(base_dir + pattern)
        match = []
        for server_path in unfiltered_match:
            real_path = os.path.abspath(server_path)
            if not valid_user_path(real_path, base_dir, True):

                # out of bounds - save user warning for later to allow
                # partial match:
                # ../*/* is technically allowed to match own files.

                logger.warning('%s tried to %s restricted path %s ! (%s)'
                               % (client_id, op_name, real_path, pattern))
                continue
            match.append(real_path)

        # Now actually treat list of allowed matchings and notify if no
        # (allowed) match

        if not match:
            output_objects.append({'object_type': 'file_not_found',
                                  'name': pattern})
            status = returnvalues.FILE_NOT_FOUND

        for real_path in match:
            relative_path = real_path.replace(base_dir, '')
            output_lines = []
            try:
                filedes = open(real_path, 'r')
                i = 0
                for line in filedes:
                    if i >= lines:
                        break
                    output_lines.append(line)
                    i += 1
                filedes.close()
            except Exception, exc:
                output_objects.append({'object_type': 'error_text',
                        'text': "%s: '%s': %s" % (op_name,
                        relative_path, exc)})
                logger.error("%s: failed on '%s': %s" % (op_name,
                             relative_path, exc))
                status = returnvalues.SYSTEM_ERROR
                continue
            entry = {'object_type': 'file_output',
                       'lines': output_lines,
                       'wrap_binary': binary(flags),
                       'wrap_targets': ['lines']}
            if verbose(flags):
                entry['path'] = relative_path
            output_objects.append(entry)
示例#3
0
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id)
    client_dir = client_id_dir(client_id)
    status = returnvalues.OK
    defaults = signature()[1]
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
    )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    flags = ''.join(accepted['flags'])
    patterns = accepted['path']
    search = accepted['pattern'][-1]

    # Please note that base_dir must end in slash to avoid access to other
    # user dirs when own name is a prefix of another user name

    base_dir = os.path.abspath(
        os.path.join(configuration.user_home, client_dir)) + os.sep

    if verbose(flags):
        for flag in flags:
            output_objects.append({
                'object_type': 'text',
                'text': '%s using flag: %s' % (op_name, flag)
            })

    for pattern in patterns:

        # Check directory traversal attempts before actual handling to avoid
        # leaking information about file system layout while allowing
        # consistent error messages

        unfiltered_match = glob.glob(base_dir + pattern)
        match = []
        for server_path in unfiltered_match:
            # IMPORTANT: path must be expanded to abs for proper chrooting
            abs_path = os.path.abspath(server_path)
            if not valid_user_path(configuration, abs_path, base_dir, True):

                # out of bounds - save user warning for later to allow
                # partial match:
                # ../*/* is technically allowed to match own files.

                logger.warning('%s tried to %s restricted path %s! (%s)' %
                               (client_id, op_name, abs_path, pattern))
                continue
            match.append(abs_path)

        # Now actually treat list of allowed matchings and notify if no
        # (allowed) match

        if not match:
            output_objects.append({
                'object_type': 'file_not_found',
                'name': pattern
            })
            status = returnvalues.FILE_NOT_FOUND

        for abs_path in match:
            relative_path = abs_path.replace(base_dir, '')
            output_lines = []
            try:
                matching = pattern_match_file(search, abs_path)
                for line in matching:
                    output_lines.append(line)
            except Exception, exc:
                output_objects.append({
                    'object_type':
                    'error_text',
                    'text':
                    "%s: '%s': %s" % (op_name, relative_path, exc)
                })
                logger.error("%s: failed on '%s': %s" %
                             (op_name, relative_path, exc))
                status = returnvalues.SYSTEM_ERROR
                continue
            entry = {
                'object_type': 'file_output',
                'lines': output_lines,
                'wrap_binary': binary(flags),
                'wrap_targets': ['lines']
            }
            if verbose(flags):
                entry['path'] = relative_path
            output_objects.append(entry)
示例#4
0
文件: tail.py 项目: heromod/migrid
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id)
    client_dir = client_id_dir(client_id)
    status = returnvalues.OK
    defaults = signature()[1]
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
        )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    flags = ''.join(accepted['flags'])
    lines = int(accepted['lines'][-1])
    pattern_list = accepted['path']

    # Please note that base_dir must end in slash to avoid access to other
    # user dirs when own name is a prefix of another user name

    base_dir = os.path.abspath(os.path.join(configuration.user_home,
                               client_dir)) + os.sep

    if verbose(flags):
        for flag in flags:
            output_objects.append({'object_type': 'text', 'text'
                                  : '%s using flag: %s' % (op_name,
                                  flag)})

    for pattern in pattern_list:

        # Check directory traversal attempts before actual handling to avoid
        # leaking information about file system layout while allowing
        # consistent error messages

        unfiltered_match = glob.glob(base_dir + pattern)
        match = []
        for server_path in unfiltered_match:
            real_path = os.path.abspath(server_path)
            if not valid_user_path(real_path, base_dir, True):

                # out of bounds - save user warning for later to allow
                # partial match:
                # ../*/* is technically allowed to match own files.

                logger.warning('%s tried to %s restricted path %s ! (%s)'
                               % (client_id, op_name, real_path, pattern))
                continue
            match.append(real_path)

        # Now actually treat list of allowed matchings and notify if no
        # (allowed) match

        if not match:
            output_objects.append({'object_type': 'file_not_found',
                                  'name': pattern})
            status = returnvalues.FILE_NOT_FOUND

        for real_path in match:
            relative_path = real_path.replace(base_dir, '')
            output_lines = []

            # We search for the last 'lines' lines by beginning from the end of
            # the file and exponetially backtracking until the backtrack
            # contains at least 'lines' lines or we're back to the beginning of
            # the file.
            # At that point we skip any extra lines before printing.

            try:
                filedes = open(real_path, 'r')

                # Go to end of file and backtrack

                backstep = 1
                newlines = 0
                filedes.seek(0, 2)
                length = filedes.tell()
                while backstep < length and newlines <= lines:
                    backstep *= 2
                    if backstep > length:
                        backstep = length
                    filedes.seek(-backstep, 2)
                    newlines = len(filedes.readlines())
                if length:

                    # Go back after reading to end of file

                    filedes.seek(-backstep, 2)

                # Skip any extra lines caused by the exponential backtracking.
                # We could probably speed up convergence with binary search...

                while newlines > lines:
                    dummy = filedes.readline()
                    newlines -= 1
                backstep = length - filedes.tell()

                # Now we're at the wanted spot - print rest of file

                for _ in range(newlines):
                    output_lines.append(filedes.readline())
                filedes.close()
            except Exception, exc:
                output_objects.append({'object_type': 'error_text',
                        'text': "%s: '%s': %s" % (op_name,
                        relative_path, exc)})
                logger.error("%s: failed on '%s': %s" % (op_name,
                             relative_path, exc))
                status = returnvalues.SYSTEM_ERROR
                continue
            entry = {'object_type': 'file_output',
                       'lines': output_lines,
                       'wrap_binary': binary(flags),
                       'wrap_targets': ['lines']}
            if verbose(flags):
                entry['path'] = relative_path
            output_objects.append(entry)
示例#5
0
文件: cat.py 项目: ucphhpc/migrid
def main(client_id, user_arguments_dict, environ=None):
    """Main function used by front end"""

    if environ is None:
        environ = os.environ

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id)
    client_dir = client_id_dir(client_id)
    defaults = signature()[1]
    status = returnvalues.OK
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
    )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    flags = ''.join(accepted['flags'])
    patterns = accepted['path']
    dst = accepted['dst'][-1].lstrip(os.sep)

    # Please note that base_dir must end in slash to avoid access to other
    # user dirs when own name is a prefix of another user name

    base_dir = os.path.abspath(
        os.path.join(configuration.user_home, client_dir)) + os.sep

    if verbose(flags):
        for flag in flags:
            output_objects.append({
                'object_type': 'text',
                'text': '%s using flag: %s' % (op_name, flag)
            })
    if dst:
        if not safe_handler(configuration, 'post', op_name, client_id,
                            get_csrf_limit(configuration), accepted):
            output_objects.append({
                'object_type':
                'error_text',
                'text':
                '''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
            })
            return (output_objects, returnvalues.CLIENT_ERROR)

        dst_mode = "wb"
        # IMPORTANT: path must be expanded to abs for proper chrooting
        abs_dest = os.path.abspath(os.path.join(base_dir, dst))
        relative_dst = abs_dest.replace(base_dir, '')
        if not valid_user_path(configuration, abs_dest, base_dir, True):
            logger.warning('%s tried to %s into restricted path %s ! (%s)' %
                           (client_id, op_name, abs_dest, dst))
            output_objects.append({
                'object_type': 'error_text',
                'text': "invalid destination: '%s'" % dst
            })
            return (output_objects, returnvalues.CLIENT_ERROR)

    for pattern in patterns:

        # Check directory traversal attempts before actual handling to avoid
        # leaking information about file system layout while allowing
        # consistent error messages

        unfiltered_match = glob.glob(base_dir + pattern)
        match = []
        for server_path in unfiltered_match:
            # IMPORTANT: path must be expanded to abs for proper chrooting
            abs_path = os.path.abspath(server_path)
            if not valid_user_path(configuration, abs_path, base_dir, True):

                # out of bounds - save user warning for later to allow
                # partial match:
                # ../*/* is technically allowed to match own files.

                logger.warning('%s tried to %s restricted path %s ! (%s)' %
                               (client_id, op_name, abs_path, pattern))
                continue
            match.append(abs_path)

        # Now actually treat list of allowed matchings and notify if no
        # (allowed) match

        if not match:
            output_objects.append({
                'object_type': 'file_not_found',
                'name': pattern
            })
            status = returnvalues.FILE_NOT_FOUND

        for abs_path in match:
            output_lines = []
            relative_path = abs_path.replace(base_dir, '')
            try:
                gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
                          'accessed', [relative_path])
                fd = open(abs_path, 'r')

                # use file directly as iterator for efficiency

                for line in fd:
                    output_lines.append(line)
                fd.close()
            except Exception, exc:
                if not isinstance(exc, GDPIOLogError):
                    gdp_iolog(configuration,
                              client_id,
                              environ['REMOTE_ADDR'],
                              'accessed', [relative_path],
                              failed=True,
                              details=exc)
                output_objects.append({
                    'object_type':
                    'error_text',
                    'text':
                    "%s: '%s': %s" % (op_name, relative_path, exc)
                })
                logger.error("%s: failed on '%s': %s" %
                             (op_name, relative_path, exc))

                status = returnvalues.SYSTEM_ERROR
                continue
            if dst:
                try:
                    gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
                              'modified', [dst])
                    out_fd = open(abs_dest, dst_mode)
                    out_fd.writelines(output_lines)
                    out_fd.close()
                    logger.info('%s %s %s done' %
                                (op_name, abs_path, abs_dest))
                except Exception, exc:
                    if not isinstance(exc, GDPIOLogError):
                        gdp_iolog(configuration,
                                  client_id,
                                  environ['REMOTE_ADDR'],
                                  'modified', [dst],
                                  error=True,
                                  details=exc)
                    output_objects.append({
                        'object_type': 'error_text',
                        'text': "write failed: '%s'" % exc
                    })
                    logger.error("%s: write failed on '%s': %s" %
                                 (op_name, abs_dest, exc))
                    status = returnvalues.SYSTEM_ERROR
                    continue
                output_objects.append({
                    'object_type':
                    'text',
                    'text':
                    "wrote %s to %s" % (relative_path, relative_dst)
                })
                # Prevent truncate after first write
                dst_mode = "ab+"
            else:
                entry = {
                    'object_type': 'file_output',
                    'lines': output_lines,
                    'wrap_binary': binary(flags),
                    'wrap_targets': ['lines']
                }
                if verbose(flags):
                    entry['path'] = relative_path
                output_objects.append(entry)

                # TODO: rip this hack out into real download handler?
                # Force download of files when output_format == 'file_format'
                # This will only work for the first file matching a glob when
                # using file_format.
                # And it is supposed to only work for one file.
                if user_arguments_dict.has_key('output_format'):
                    output_format = user_arguments_dict['output_format'][0]
                    if output_format == 'file':
                        output_objects.append({
                            'object_type':
                            'start',
                            'headers': [('Content-Disposition',
                                         'attachment; filename="%s";' %
                                         os.path.basename(abs_path))]
                        })
示例#6
0
文件: cat.py 项目: heromod/migrid
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id)
    client_dir = client_id_dir(client_id)
    defaults = signature()[1]
    status = returnvalues.OK
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
        )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    flags = ''.join(accepted['flags'])
    patterns = accepted['path']
    dst = accepted['dst'][-1]

    # Please note that base_dir must end in slash to avoid access to other
    # user dirs when own name is a prefix of another user name

    base_dir = os.path.abspath(os.path.join(configuration.user_home,
                               client_dir)) + os.sep

    if verbose(flags):
        for flag in flags:
            output_objects.append({'object_type': 'text', 'text'
                                  : '%s using flag: %s' % (op_name,
                                  flag)})

    if dst:
        dst_mode = "wb"
        real_dst = os.path.join(base_dir, dst)
        relative_dst = real_dst.replace(base_dir, '')
        if not valid_user_path(real_dst, base_dir, True):
            logger.warning('%s tried to %s into restricted path %s ! (%s)'
                           % (client_id, op_name, real_dst, dst))
            output_objects.append({'object_type': 'error_text',
                                   'text': "invalid destination: '%s'" % \
                                   dst})
            return (output_objects, returnvalues.CLIENT_ERROR)
            
    for pattern in patterns:

        # Check directory traversal attempts before actual handling to avoid
        # leaking information about file system layout while allowing
        # consistent error messages

        unfiltered_match = glob.glob(base_dir + pattern)
        match = []
        for server_path in unfiltered_match:
            real_path = os.path.abspath(server_path)
            if not valid_user_path(real_path, base_dir, True):

                # out of bounds - save user warning for later to allow
                # partial match:
                # ../*/* is technically allowed to match own files.

                logger.warning('%s tried to %s restricted path %s ! (%s)'
                               % (client_id, op_name, real_path, pattern))
                continue
            match.append(real_path)

        # Now actually treat list of allowed matchings and notify if no
        # (allowed) match

        if not match:
            output_objects.append({'object_type': 'file_not_found',
                                  'name': pattern})
            status = returnvalues.FILE_NOT_FOUND

        for real_path in match:
            output_lines = []
            relative_path = real_path.replace(base_dir, '')
            try:
                fd = open(real_path, 'r')

                # use file directly as iterator for efficiency

                for line in fd:
                    output_lines.append(line)
                fd.close()
            except Exception, exc:
                output_objects.append({'object_type': 'error_text',
                        'text': "%s: '%s': %s" % (op_name,
                        relative_path, exc)})
                logger.error("%s: failed on '%s': %s" % (op_name,
                             relative_path, exc))
                status = returnvalues.SYSTEM_ERROR
                continue
            if dst:
                try:
                    out_fd = open(real_dst, dst_mode)
                    out_fd.writelines(output_lines)
                    out_fd.close()
                except Exception, exc:
                    output_objects.append({'object_type': 'error_text',
                                           'text': "write failed: '%s'" % exc})
                    logger.error("%s: write failed on '%s': %s" % (op_name,
                                                               real_dst, exc))
                    status = returnvalues.SYSTEM_ERROR
                    continue
                output_objects.append({'object_type': 'text',
                        'text': "wrote %s to %s" % (relative_path,
                                                    relative_dst)})
                # Prevent truncate after first write
                dst_mode = "ab+"
            else:
                entry = {'object_type': 'file_output',
                         'lines': output_lines,
                         'wrap_binary': binary(flags),
                         'wrap_targets': ['lines']}
                if verbose(flags):
                    entry['path'] = relative_path
                output_objects.append(entry)

                # TODO: rip this hack out into real download handler?
                # Force download of files when output_format == 'file_format'
                # This will only work for the first file matching a glob when
                # using file_format.
                # And it is supposed to only work for one file.
                if user_arguments_dict.has_key('output_format'):
                    output_format = user_arguments_dict['output_format'][0]
                    if output_format == 'file':
                        output_objects.append(
                            {'object_type': 'start', 'headers':
                             [('Content-Disposition',
                               'attachment; filename="%s";' % \
                               os.path.basename(real_path))]})