def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') output_lines = [] # We search for the last 'lines' lines by beginning from the end of # the file and exponetially backtracking until the backtrack # contains at least 'lines' lines or we're back to the beginning of # the file. # At that point we skip any extra lines before printing. try: filedes = open(abs_path, 'r') # Go to end of file and backtrack backstep = 1 newlines = 0 filedes.seek(0, 2) length = filedes.tell() while backstep < length and newlines <= lines: backstep *= 2 if backstep > length: backstep = length filedes.seek(-backstep, 2) newlines = len(filedes.readlines()) if length: # Go back after reading to end of file filedes.seek(-backstep, 2) # Skip any extra lines caused by the exponential backtracking. # We could probably speed up convergence with binary search... while newlines > lines: dummy = filedes.readline() newlines -= 1 backstep = length - filedes.tell() # Now we're at the wanted spot - print rest of file for _ in range(newlines): output_lines.append(filedes.readline()) filedes.close() except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = { 'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines'] } if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') output_lines = [] try: filedes = open(real_path, 'r') i = 0 for line in filedes: if i >= lines: break output_lines.append(line) i += 1 filedes.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = {'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines']} if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] search = accepted['pattern'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') output_lines = [] try: matching = pattern_match_file(search, abs_path) for line in matching: output_lines.append(line) except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = { 'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines'] } if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') output_lines = [] # We search for the last 'lines' lines by beginning from the end of # the file and exponetially backtracking until the backtrack # contains at least 'lines' lines or we're back to the beginning of # the file. # At that point we skip any extra lines before printing. try: filedes = open(real_path, 'r') # Go to end of file and backtrack backstep = 1 newlines = 0 filedes.seek(0, 2) length = filedes.tell() while backstep < length and newlines <= lines: backstep *= 2 if backstep > length: backstep = length filedes.seek(-backstep, 2) newlines = len(filedes.readlines()) if length: # Go back after reading to end of file filedes.seek(-backstep, 2) # Skip any extra lines caused by the exponential backtracking. # We could probably speed up convergence with binary search... while newlines > lines: dummy = filedes.readline() newlines -= 1 backstep = length - filedes.tell() # Now we're at the wanted spot - print rest of file for _ in range(newlines): output_lines.append(filedes.readline()) filedes.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = {'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines']} if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] status = returnvalues.OK (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] dst = accepted['dst'][-1].lstrip(os.sep) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) if dst: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) dst_mode = "wb" # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) relative_dst = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): logger.warning('%s tried to %s into restricted path %s ! (%s)' % (client_id, op_name, abs_dest, dst)) output_objects.append({ 'object_type': 'error_text', 'text': "invalid destination: '%s'" % dst }) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'accessed', [relative_path]) fd = open(abs_path, 'r') # use file directly as iterator for efficiency for line in fd: output_lines.append(line) fd.close() except Exception, exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'accessed', [relative_path], failed=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if dst: try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'modified', [dst]) out_fd = open(abs_dest, dst_mode) out_fd.writelines(output_lines) out_fd.close() logger.info('%s %s %s done' % (op_name, abs_path, abs_dest)) except Exception, exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'modified', [dst], error=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "write failed: '%s'" % exc }) logger.error("%s: write failed on '%s': %s" % (op_name, abs_dest, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({ 'object_type': 'text', 'text': "wrote %s to %s" % (relative_path, relative_dst) }) # Prevent truncate after first write dst_mode = "ab+" else: entry = { 'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines'] } if verbose(flags): entry['path'] = relative_path output_objects.append(entry) # TODO: rip this hack out into real download handler? # Force download of files when output_format == 'file_format' # This will only work for the first file matching a glob when # using file_format. # And it is supposed to only work for one file. if user_arguments_dict.has_key('output_format'): output_format = user_arguments_dict['output_format'][0] if output_format == 'file': output_objects.append({ 'object_type': 'start', 'headers': [('Content-Disposition', 'attachment; filename="%s";' % os.path.basename(abs_path))] })
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] status = returnvalues.OK (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] dst = accepted['dst'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) if dst: dst_mode = "wb" real_dst = os.path.join(base_dir, dst) relative_dst = real_dst.replace(base_dir, '') if not valid_user_path(real_dst, base_dir, True): logger.warning('%s tried to %s into restricted path %s ! (%s)' % (client_id, op_name, real_dst, dst)) output_objects.append({'object_type': 'error_text', 'text': "invalid destination: '%s'" % \ dst}) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: output_lines = [] relative_path = real_path.replace(base_dir, '') try: fd = open(real_path, 'r') # use file directly as iterator for efficiency for line in fd: output_lines.append(line) fd.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if dst: try: out_fd = open(real_dst, dst_mode) out_fd.writelines(output_lines) out_fd.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "write failed: '%s'" % exc}) logger.error("%s: write failed on '%s': %s" % (op_name, real_dst, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'text', 'text': "wrote %s to %s" % (relative_path, relative_dst)}) # Prevent truncate after first write dst_mode = "ab+" else: entry = {'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines']} if verbose(flags): entry['path'] = relative_path output_objects.append(entry) # TODO: rip this hack out into real download handler? # Force download of files when output_format == 'file_format' # This will only work for the first file matching a glob when # using file_format. # And it is supposed to only work for one file. if user_arguments_dict.has_key('output_format'): output_format = user_arguments_dict['output_format'][0] if output_format == 'file': output_objects.append( {'object_type': 'start', 'headers': [('Content-Disposition', 'attachment; filename="%s";' % \ os.path.basename(real_path))]})