def read_trigger_log(configuration, vgrid_name, flags): """Read-in saved trigger logs for vgrid workflows page. We read in all rotated logs. If flags don't include verbose we try to filter out all system trigger lines. """ log_content = '' for i in xrange(workflows_log_cnt - 1, -1, -1): log_name = '%s.%s' % (configuration.vgrid_triggers, workflows_log_name) if i > 0: log_name += '.%d' % i log_path = os.path.join(configuration.vgrid_home, vgrid_name, log_name) configuration.logger.debug('read from %s' % log_path) try: log_fd = open(log_path) log_content += log_fd.read() configuration.logger.debug('read in log lines:\n%s' % log_content) log_fd.close() except IOError: pass if not verbose(flags): # Strip system trigger lines containing '.meta/EXT.last_modified' system_pattern = '[0-9 ,:-]* [A-Z]* .*/\.meta/.*\.last_modified.*\n' log_content = re.sub(system_pattern, '', log_content) return log_content
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] name_pattern = accepted['name'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) dir_listings = [] output_objects.append({ 'object_type': 'dir_listings', 'dir_listings': dir_listings, 'flags': flags, }) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') entries = [] dir_listing = { 'object_type': 'dir_listing', 'relative_path': relative_path, 'entries': entries, 'flags': flags, } dir_listings.append(dir_listing) try: for (root, dirs, files) in os.walk(abs_path): for filename in fnmatch.filter(files, name_pattern): # IMPORTANT: this join always yields abs expanded path abs_path = os.path.join(root, filename) relative_path = abs_path.replace(base_dir, '') if not valid_user_path(configuration, abs_path, base_dir, True): continue file_obj = { 'object_type': 'direntry', 'type': 'file', 'name': filename, 'file_with_dir': relative_path, 'flags': flags, 'special': '', } entries.append(file_obj) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if verbose(flags): output_objects.append({'object_type': 'file_output', 'path': relative_path, 'lines': output_lines}) else: output_objects.append({'object_type': 'file_output', 'lines': output_lines})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] current_dir = accepted['current_dir'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages # NB: Globbing disabled on purpose here unfiltered_match = [base_dir + pattern] match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append( {'object_type': 'error_text', 'text' : "%s: cannot remove directory '%s': Permission denied" % (op_name, pattern)}) status = returnvalues.CLIENT_ERROR for real_path in match: relative_path = real_path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file', 'name' : relative_path}) if not os.path.exists(real_path): output_objects.append({'object_type': 'file_not_found', 'name': relative_path}) continue try: if parents(flags): # Please note that 'rmdir -p X' should give error if X # doesn't exist contrary to 'mkdir -p X' not giving error # if X exists. os.removedirs(real_path) else: os.rmdir(real_path) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s failed on '%s'" % (op_name, relative_path)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) size = int(accepted['size'][-1]) pattern_list = accepted['path'] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) if size < 0: output_objects.append({ 'object_type': 'error_text', 'text': 'size must be non-negative' }) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') if verbose(flags): output_objects.append({ 'object_type': 'file', 'name': relative_path }) if not check_write_access(abs_path): logger.warning('%s called without write access: %s' % \ (op_name, abs_path)) output_objects.append( {'object_type': 'error_text', 'text': 'cannot truncate "%s": inside a read-only location!' % \ pattern}) status = returnvalues.CLIENT_ERROR continue try: fd = open(abs_path, 'r+') fd.truncate(size) fd.close() logger.info('%s %s %s done' % (op_name, abs_path, size)) except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue
dst_dir = cache_dir title_entry = find_entry(output_objects, 'title') title_entry['text'] = header_item['text'] = page_title title_entry['skipwidgets'] = not widgets title_entry['skipuserstyle'] = not userstyle # Input validation assures target_dir can't escape base_dir if not os.path.isdir(base_dir): output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid client/sharelink id!' }) return (output_objects, returnvalues.CLIENT_ERROR) if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) logger.info('parsing upload form in %s' % op_name) # Now parse and validate files to archive # ... this includes checking for illegal directory traversal attempts for name in defaults.keys(): if user_arguments_dict.has_key(name): del user_arguments_dict[name]
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) dst = accepted['dst'][-1].lstrip(os.sep) pattern_list = accepted['src'] current_dir = accepted['current_dir'][-1].lstrip(os.sep) # All paths are relative to current_dir pattern_list = [os.path.join(current_dir, i) for i in pattern_list] dst = os.path.join(current_dir, dst) if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append( {'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'ZIP/TAR archiver' output_objects.append( {'object_type': 'header', 'text': 'ZIP/TAR archiver'}) if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag)}) if 'h' in flags: usage(output_objects) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dir = os.path.abspath(os.path.join(base_dir, current_dir.lstrip(os.sep))) if not valid_user_path(configuration, abs_dir, base_dir, True): # out of bounds output_objects.append({'object_type': 'error_text', 'text': "You're not allowed to work in %s!" % current_dir}) logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dir, current_dir)) return (output_objects, returnvalues.CLIENT_ERROR) output_objects.append( {'object_type': 'text', 'text': "working in %s" % current_dir}) # NOTE: dst already incorporates current_dir prefix here # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) logger.info('pack in %s' % abs_dest) # Don't use abs_path in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): # out of bounds output_objects.append( {'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % dst}) logger.warning('%s tried to %s restricted path %s !(%s)' % (client_id, op_name, abs_dest, dst)) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(os.path.dirname(abs_dest)): output_objects.append({'object_type': 'error_text', 'text': "No such destination directory: %s" % os.path.dirname(relative_dest)}) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_dest)) output_objects.append( {'object_type': 'error_text', 'text': 'cannot pack to "%s": inside a read-only location!' % relative_dest}) return (output_objects, returnvalues.CLIENT_ERROR) status = returnvalues.OK for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: logger.debug("%s: inspecting: %s" % (op_name, server_path)) # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue else: match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'error_text', 'text': "%s: cannot pack '%s': no valid src paths" % (op_name, pattern)}) status = returnvalues.CLIENT_ERROR continue for abs_path in match: relative_path = abs_path.replace(base_dir, '') # Generally refuse handling symlinks including root vgrid shares if os.path.islink(abs_path): output_objects.append( {'object_type': 'warning', 'text': """You're not allowed to pack entire special folders like %s shared folders!""" % configuration.site_vgrid_label}) status = returnvalues.CLIENT_ERROR continue # Additionally refuse operations on inherited subvgrid share roots elif in_vgrid_share(configuration, abs_path) == relative_path: output_objects.append( {'object_type': 'warning', 'text': """You're not allowed to pack entire %s shared folders!""" % configuration.site_vgrid_label}) status = returnvalues.CLIENT_ERROR continue elif os.path.realpath(abs_path) == os.path.realpath(base_dir): logger.error("%s: refusing pack home dir: %s" % (op_name, abs_path)) output_objects.append( {'object_type': 'warning', 'text': "You're not allowed to pack your entire home directory!" }) status = returnvalues.CLIENT_ERROR continue elif os.path.realpath(abs_dest) == os.path.realpath(abs_path): output_objects.append({'object_type': 'warning', 'text': 'overlap in source and destination %s' % relative_dest}) status = returnvalues.CLIENT_ERROR continue if verbose(flags): output_objects.append( {'object_type': 'file', 'name': relative_path}) (pack_status, msg) = pack_archive(configuration, client_id, relative_path, relative_dest) if not pack_status: output_objects.append({'object_type': 'error_text', 'text': 'Error: %s' % msg}) status = returnvalues.CLIENT_ERROR continue output_objects.append({'object_type': 'text', 'text': 'Added %s to %s' % (relative_path, relative_dest)}) output_objects.append({'object_type': 'text', 'text': 'Packed archive of %s is now available in %s' % (', '.join(pattern_list), relative_dest)}) output_objects.append({'object_type': 'link', 'text': 'Download archive', 'destination': os.path.join('..', client_dir, relative_dest)}) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') output_lines = [] # We search for the last 'lines' lines by beginning from the end of # the file and exponetially backtracking until the backtrack # contains at least 'lines' lines or we're back to the beginning of # the file. # At that point we skip any extra lines before printing. try: filedes = open(real_path, 'r') # Go to end of file and backtrack backstep = 1 newlines = 0 filedes.seek(0, 2) length = filedes.tell() while backstep < length and newlines <= lines: backstep *= 2 if backstep > length: backstep = length filedes.seek(-backstep, 2) newlines = len(filedes.readlines()) if length: # Go back after reading to end of file filedes.seek(-backstep, 2) # Skip any extra lines caused by the exponential backtracking. # We could probably speed up convergence with binary search... while newlines > lines: dummy = filedes.readline() newlines -= 1 backstep = length - filedes.tell() # Now we're at the wanted spot - print rest of file for _ in range(newlines): output_lines.append(filedes.readline()) filedes.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = {'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines']} if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input(user_arguments_dict, defaults, output_objects, allow_rejects=False) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) pattern_list = accepted['path'] iosessionid = accepted['iosessionid'][-1] if not client_id: if not iosessionid.strip() or not iosessionid.isalnum(): # deny output_objects.append( {'object_type': 'error_text', 'text' : 'No sessionid or invalid sessionid supplied!'}) return (output_objects, returnvalues.CLIENT_ERROR) base_dir_no_sessionid = \ os.path.realpath(configuration.webserver_home) + os.sep base_dir = \ os.path.realpath(os.path.join(configuration.webserver_home, iosessionid)) + os.sep if not os.path.isdir(base_dir): # deny output_objects.append({'object_type': 'error_text', 'text' : 'Invalid sessionid!'}) return (output_objects, returnvalues.CLIENT_ERROR) if not valid_user_path(base_dir, base_dir_no_sessionid, True): # deny output_objects.append({'object_type': 'error_text', 'text' : 'Invalid sessionid!'}) return (output_objects, returnvalues.CLIENT_ERROR) else: # TODO: this is a hack to allow truncate - fix 'put' empty files # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! ( %s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file', 'name' : relative_path}) # Make it harder to accidentially delete too much - e.g. do not delete # VGrid files without explicit selection of subdir contents if real_path == os.path.abspath(base_dir): output_objects.append({'object_type': 'warning', 'text' : "You're not allowed to delete your entire home directory!" }) status = returnvalues.CLIENT_ERROR continue if os.path.islink(real_path): output_objects.append({'object_type': 'warning', 'text' : "You're not allowed to delete entire %s shared dirs!" % configuration.site_vgrid_label }) status = returnvalues.CLIENT_ERROR continue if os.path.isdir(real_path) and recursive(flags): # bottom up traversal of the file tree since rmdir is limited to # empty dirs for (root, dirs, files) in os.walk(real_path, topdown=False): for name in files: path = os.path.join(root, name) relative_path = path.replace(base_dir, '') # Traversal may find additional invisible files to skip if invisible_file(name): continue if verbose(flags): output_objects.append({'object_type': 'file' , 'name': relative_path}) try: os.remove(path) except Exception, exc: output_objects.append({'object_type' : 'error_text', 'text' : "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR for name in dirs: path = os.path.join(root, name) relative_path = path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file' , 'name': relative_path}) try: os.rmdir(path) except Exception, exc: output_objects.append({'object_type' : 'error_text', 'text' : "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR # Finally remove base directory relative_path = real_path.replace(base_dir, '') try: os.rmdir(real_path) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) dst = accepted['dst'][-1].lstrip(os.sep) pattern_list = accepted['src'] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append( {'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'Zip/tar archive extractor' output_objects.append({'object_type': 'header', 'text' : 'Zip/tar archive extractor'}) if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) if 'h' in flags: usage(output_objects) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) logger.info('unpack in %s' % abs_dest) # Don't use real dest in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): # out of bounds logger.error('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dest, dst)) output_objects.append( {'object_type': 'error_text', 'text' : "Invalid path! (%s expands to an illegal path)" % dst}) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % \ (op_name, abs_dest)) output_objects.append( {'object_type': 'error_text', 'text': 'cannot unpack to "%s": inside a read-only location!' % \ relative_dest}) return (output_objects, returnvalues.CLIENT_ERROR) status = returnvalues.OK for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file', 'name' : relative_path}) (unpack_status, msg) = unpack_archive(configuration, client_id, relative_path, relative_dest) if not unpack_status: output_objects.append({'object_type': 'error_text', 'text': 'Error: %s' % msg}) status = returnvalues.CLIENT_ERROR continue output_objects.append( {'object_type': 'text', 'text' : 'The zip/tar archive %s was unpacked in %s' % (relative_path, relative_dest)}) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] status = returnvalues.OK (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] dst = accepted['dst'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) if dst: dst_mode = "wb" real_dst = os.path.join(base_dir, dst) relative_dst = real_dst.replace(base_dir, '') if not valid_user_path(real_dst, base_dir, True): logger.warning('%s tried to %s into restricted path %s ! (%s)' % (client_id, op_name, real_dst, dst)) output_objects.append({'object_type': 'error_text', 'text': "invalid destination: '%s'" % \ dst}) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: output_lines = [] relative_path = real_path.replace(base_dir, '') try: fd = open(real_path, 'r') # use file directly as iterator for efficiency for line in fd: output_lines.append(line) fd.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if dst: try: out_fd = open(real_dst, dst_mode) out_fd.writelines(output_lines) out_fd.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "write failed: '%s'" % exc}) logger.error("%s: write failed on '%s': %s" % (op_name, real_dst, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'text', 'text': "wrote %s to %s" % (relative_path, relative_dst)}) # Prevent truncate after first write dst_mode = "ab+" else: entry = {'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines']} if verbose(flags): entry['path'] = relative_path output_objects.append(entry) # TODO: rip this hack out into real download handler? # Force download of files when output_format == 'file_format' # This will only work for the first file matching a glob when # using file_format. # And it is supposed to only work for one file. if user_arguments_dict.has_key('output_format'): output_format = user_arguments_dict['output_format'][0] if output_format == 'file': output_objects.append( {'object_type': 'start', 'headers': [('Content-Disposition', 'attachment; filename="%s";' % \ os.path.basename(real_path))]})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) src_list = accepted['src'] dst = accepted['dst'][-1] iosessionid = accepted['iosessionid'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if not client_id: base_dir = os.path.realpath(configuration.webserver_home + os.sep + iosessionid) + os.sep status = returnvalues.OK real_dest = base_dir + dst dst_list = glob.glob(real_dest) if not dst_list: # New destination? if not glob.glob(os.path.dirname(real_dest)): output_objects.append({'object_type': 'error_text', 'text' : 'Illegal dst path provided!'}) return (output_objects, returnvalues.CLIENT_ERROR) else: dst_list = [real_dest] # Use last match in case of multiple matches dest = dst_list[-1] if len(dst_list) > 1: output_objects.append( {'object_type': 'warning', 'text' : 'dst (%s) matches multiple targets - using last: %s' % (dst, dest)}) real_dest = os.path.abspath(dest) # Don't use real_path in output as it may expose underlying # fs layout. relative_dest = real_dest.replace(base_dir, '') if not valid_user_path(real_dest, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_dest, dst)) output_objects.append( {'object_type': 'error_text', 'text' : "Invalid destination (%s expands to an illegal path)" % dst}) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in src_list: unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file', 'name' : relative_path}) # src must be a file unless recursive is specified if not recursive(flags) and os.path.isdir(real_path): logger.warning('skipping directory source %s' % real_path) output_objects.append({'object_type': 'warning', 'text' : 'skipping directory src %s!' % relative_path}) continue # If destination is a directory the src should be copied there real_target = real_dest if os.path.isdir(real_target): real_target = os.path.join(real_target, os.path.basename(real_path)) if os.path.abspath(real_path) == os.path.abspath(real_target): logger.warning('%s tried to %s %s to itself! (%s)' % \ (client_id, op_name, real_path, pattern)) output_objects.append( {'object_type': 'warning', 'text' : "Cannot copy '%s' to self!" % relative_path}) status = returnvalues.CLIENT_ERROR continue if os.path.isdir(real_path) and \ real_target.startswith(real_path + os.sep): logger.warning('%s tried to %s %s to itself! (%s)' % (client_id, op_name, real_path, pattern)) output_objects.append( {'object_type': 'warning', 'text' : "Cannot copy '%s' to (sub) self!" % relative_path}) status = returnvalues.CLIENT_ERROR continue try: if os.path.isdir(real_path): shutil.copytree(real_path, real_target) else: shutil.copy(real_path, real_target) logger.info('%s %s %s done' % (op_name, real_path, real_target)) except Exception, exc: output_objects.append( {'object_type': 'error_text', 'text': "%s: failed on '%s' to '%s'" \ % (op_name, relative_path, relative_dest)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag)}) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND # NOTE: we produce output matching an invocation of: # du -aL --apparent-size --block-size=1 PATH [PATH ...] filedus = [] summarize_output = summarize(flags) for abs_path in match: if invisible_path(abs_path): continue relative_path = abs_path.replace(base_dir, '') # cache accumulated sub dir sizes - du sums into parent dir size dir_sizes = {} try: # Assume a directory to walk for (root, dirs, files) in walk(abs_path, topdown=False, followlinks=True): if invisible_path(root): continue dir_bytes = 0 for name in files: real_file = os.path.join(root, name) if invisible_path(real_file): continue relative_file = real_file.replace(base_dir, '') size = os.path.getsize(real_file) dir_bytes += size if not summarize_output: filedus.append({'object_type': 'filedu', 'name': relative_file, 'bytes': size}) for name in dirs: real_dir = os.path.join(root, name) if invisible_path(real_dir): continue dir_bytes += dir_sizes[real_dir] relative_root = root.replace(base_dir, '') dir_bytes += os.path.getsize(root) dir_sizes[root] = dir_bytes if root == abs_path or not summarize_output: filedus.append({'object_type': 'filedu', 'name': relative_root, 'bytes': dir_bytes}) if os.path.isfile(abs_path): # Fall back to plain file where walk is empty size = os.path.getsize(abs_path) filedus.append({'object_type': 'filedu', 'name': relative_path, 'bytes': size}) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'filedus', 'filedus': filedus})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] name_pattern = accepted['name'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) dir_listings = [] output_objects.append({ 'object_type': 'dir_listings', 'dir_listings': dir_listings, 'flags': flags, }) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: output_lines = [] relative_path = real_path.replace(base_dir, '') entries = [] dir_listing = { 'object_type': 'dir_listing', 'relative_path': relative_path, 'entries': entries, 'flags': flags, } dir_listings.append(dir_listing) try: for (root, dirs, files) in os.walk(real_path): for filename in fnmatch.filter(files, name_pattern): real_path = os.path.join(root, filename) relative_path = real_path.replace(base_dir, '') if not valid_user_path(real_path, base_dir, True): continue file_obj = { 'object_type': 'direntry', 'type': 'file', 'name': filename, 'file_with_dir': relative_path, 'flags': flags, 'special': '', } entries.append(file_obj) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if verbose(flags): output_objects.append({'object_type': 'file_output', 'path': relative_path, 'lines': output_lines}) else: output_objects.append({'object_type': 'file_output', 'lines': output_lines})
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) src_list = accepted['src'] dst = accepted['dst'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK abs_dest = base_dir + dst dst_list = glob.glob(abs_dest) if not dst_list: # New destination? if not glob.glob(os.path.dirname(abs_dest)): output_objects.append({ 'object_type': 'error_text', 'text': 'Illegal dst path provided!' }) return (output_objects, returnvalues.CLIENT_ERROR) else: dst_list = [abs_dest] # Use last match in case of multiple matches dest = dst_list[-1] if len(dst_list) > 1: output_objects.append({ 'object_type': 'warning', 'text': 'dst (%s) matches multiple targets - using last: %s' % (dst, dest) }) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(dest) # Don't use abs_path in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): logger.warning('%s tried to %s to restricted path %s ! (%s)' % (client_id, op_name, abs_dest, dst)) output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % dst }) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_dest)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot move to "%s": inside a read-only location!' % relative_dest }) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in src_list: unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: no such file or directory! %s' % (op_name, pattern) }) status = returnvalues.CLIENT_ERROR for abs_path in match: relative_path = abs_path.replace(base_dir, '') if verbose(flags): output_objects.append({ 'object_type': 'file', 'name': relative_path }) # Generally refuse handling symlinks including root vgrid shares if os.path.islink(abs_path): output_objects.append({ 'object_type': 'warning', 'text': """You're not allowed to move entire special folders like %s shared folders!""" % configuration.site_vgrid_label }) status = returnvalues.CLIENT_ERROR continue # Additionally refuse operations on inherited subvgrid share roots elif in_vgrid_share(configuration, abs_path) == relative_path: output_objects.append({ 'object_type': 'warning', 'text': """You're not allowed to move entire %s shared folders!""" % configuration.site_vgrid_label }) status = returnvalues.CLIENT_ERROR continue elif os.path.realpath(abs_path) == os.path.realpath(base_dir): logger.error("%s: refusing move home dir: %s" % (op_name, abs_path)) output_objects.append({ 'object_type': 'warning', 'text': "You're not allowed to move your entire home directory!" }) status = returnvalues.CLIENT_ERROR continue if not check_write_access(abs_path): logger.warning('%s called without write access: %s' % (op_name, abs_path)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot move "%s": inside a read-only location!' % pattern }) status = returnvalues.CLIENT_ERROR continue # If destination is a directory the src should be moved in there # Move with existing directory as target replaces the directory! abs_target = abs_dest if os.path.isdir(abs_target): if os.path.samefile(abs_target, abs_path): output_objects.append({ 'object_type': 'warning', 'text': "Cannot move '%s' to a subdirectory of itself!" % relative_path }) status = returnvalues.CLIENT_ERROR continue abs_target = os.path.join(abs_target, os.path.basename(abs_path)) try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'moved', [relative_path, relative_dest]) shutil.move(abs_path, abs_target) logger.info('%s %s %s done' % (op_name, abs_path, abs_target)) except Exception, exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'moved', [relative_path, relative_dest], failed=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: logger.error("jobstatus input validation failed: %s" % accepted) return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) max_jobs = int(accepted['max_jobs'][-1]) order = 'unsorted ' if sorted(flags): order = 'sorted ' patterns = accepted['job_id'] project_names = accepted['project_name'] if len(project_names) > 0: for project_name in project_names: project_name_job_ids = \ get_job_ids_with_specified_project_name(client_id, project_name, configuration.mrsl_files_dir, logger) patterns.extend(project_name_job_ids) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep output_objects.append({'object_type': 'header', 'text' : '%s %s job status' % \ (configuration.short_title, order)}) if not patterns: output_objects.append({'object_type': 'error_text', 'text' : 'No job_id specified!'}) return (output_objects, returnvalues.NO_SUCH_JOB_ID) if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) if not os.path.isdir(base_dir): output_objects.append( {'object_type': 'error_text', 'text' : ('You have not been created as a user on the %s server! ' \ 'Please contact the %s team.') % \ (configuration.short_title, configuration.short_title)}) return (output_objects, returnvalues.CLIENT_ERROR) filelist = [] for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to # avoid leaking information about file system layout while # allowing consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match.... if not match: output_objects.append( {'object_type': 'error_text', 'text' : '%s: You do not have any matching job IDs!' % pattern}) status = returnvalues.CLIENT_ERROR else: filelist += match if sorted(flags): sort(filelist) if max_jobs < len(filelist): output_objects.append( {'object_type': 'text', 'text' : 'Only showing first %d of the %d matching jobs as requested' % (max_jobs, len(filelist))}) filelist = filelist[:max_jobs] # Iterate through jobs and print details for each job_list = {'object_type': 'job_list', 'jobs': []} for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') job_dict = unpickle(filepath, logger) if not job_dict: status = returnvalues.CLIENT_ERROR output_objects.append( {'object_type': 'error_text', 'text' : 'No such job: %s (could not load mRSL file %s)' % \ (job_id, filepath)}) continue # Expand any job variables before use job_dict = expand_variables(job_dict) job_obj = {'object_type': 'job', 'job_id': job_id} job_obj['status'] = job_dict['STATUS'] time_fields = [ 'VERIFIED', 'VERIFIED_TIMESTAMP', 'RECEIVED_TIMESTAMP', 'QUEUED_TIMESTAMP', 'SCHEDULE_TIMESTAMP', 'EXECUTING_TIMESTAMP', 'FINISHED_TIMESTAMP', 'FAILED_TIMESTAMP', 'CANCELED_TIMESTAMP', ] for name in time_fields: if job_dict.has_key(name): # time objects cannot be marshalled, asctime if timestamp try: job_obj[name.lower()] = time.asctime(job_dict[name]) except Exception, exc: # not a time object, just add job_obj[name.lower()] = job_dict[name] ########################################### # ARC job status retrieval on demand: # But we should _not_ update the status in the mRSL files, since # other MiG code might rely on finding only valid "MiG" states. if configuration.arc_clusters and \ job_dict.get('UNIQUE_RESOURCE_NAME', 'unset') == 'ARC' \ and job_dict['STATUS'] == 'EXECUTING': try: home = os.path.join(configuration.user_home, client_dir) arcsession = arc.Ui(home) arcstatus = arcsession.jobStatus(job_dict['EXE']) job_obj['status'] = arcstatus['status'] except arc.ARCWrapperError, err: logger.error('Error retrieving ARC job status: %s' % \ err.what()) job_obj['status'] += '(Error: ' + err.what() + ')' except arc.NoProxyError, err: logger.error('While retrieving ARC job status: %s' % \ err.what()) job_obj['status'] += '(Error: ' + err.what() + ')'
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) pattern_list = accepted['path'] lang = accepted['lang'][-1].lower() mode = accepted['mode'][-1] output_objects.append({'object_type': 'header', 'text' : '%s spell check' % configuration.short_title }) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK allowed_modes = ['none', 'url', 'email', 'sgml', 'tex'] # Include both base languages and variants allowed_langs = [ 'da', 'da_dk', 'de', 'en', 'en_gb', 'en_us', 'es', 'fi', 'fr', 'it', 'nl', 'no', 'se', ] # TODO: use path from settings file dict_path = '%s/%s' % (base_dir, '.personal_dictionary') if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) if not mode in allowed_modes: output_objects.append({'object_type': 'error_text', 'text' : 'Unsupported mode: %s' % mode}) status = returnvalues.CLIENT_ERROR if not lang in allowed_langs: output_objects.append({'object_type': 'error_text', 'text' : 'Unsupported lang: %s' % mode}) status = returnvalues.CLIENT_ERROR # Show all if no flags given if not flags: flags = 'blw' for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') output_lines = [] try: (out, err) = spellcheck(real_path, mode, lang, dict_path) if err: output_objects.append({'object_type': 'error_text', 'text': err}) for line in out: output_lines.append(line + '\n') except Exception, err: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, err)}) status = returnvalues.SYSTEM_ERROR continue if verbose(flags): output_objects.append({'object_type': 'file_output', 'path': relative_path, 'lines': output_lines}) else: output_objects.append({'object_type': 'file_output', 'lines': output_lines}) htmlform = \ ''' <form method="post" action="editor.py"> <input type="hidden" name="path" value="%s" /> <input type="submit" value="Edit file" /> </form> '''\ % relative_path output_objects.append({'object_type': 'html_form', 'text' : htmlform})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) src_list = accepted['src'] dst = accepted['dst'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK real_dest = base_dir + dst dst_list = glob.glob(real_dest) if not dst_list: # New destination? if not glob.glob(os.path.dirname(real_dest)): output_objects.append({'object_type': 'error_text', 'text' : 'Illegal dst path provided!'}) return (output_objects, returnvalues.CLIENT_ERROR) else: dst_list = [real_dest] # Use last match in case of multiple matches dest = dst_list[-1] if len(dst_list) > 1: output_objects.append( {'object_type': 'warning', 'text' : 'dst (%s) matches multiple targets - using last: %s' % (dst, dest)}) real_dest = os.path.abspath(dest) # Don't use real_path in output as it may expose underlying # fs layout. relative_dest = real_dest.replace(base_dir, '') if not valid_user_path(real_dest, base_dir, True): logger.warning('%s tried to %s to restricted path %s ! (%s)' % (client_id, op_name, real_dest, dst)) output_objects.append( {'object_type': 'error_text', 'text' : "Invalid path! (%s expands to an illegal path)" % dst}) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in src_list: unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'error_text', 'text' : '%s: no such file or directory! %s' % (op_name, pattern)}) status = returnvalues.CLIENT_ERROR for real_path in match: relative_path = real_path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file', 'name' : relative_path}) if os.path.islink(real_path): output_objects.append( {'object_type': 'warning', 'text' : "You're not allowed to move entire %s shared dirs!" % configuration.site_vgrid_label}) status = returnvalues.CLIENT_ERROR continue # If destination is a directory the src should be moved in there # Move with existing directory as target replaces the directory! real_target = real_dest if os.path.isdir(real_target): if os.path.samefile(real_target, real_path): output_objects.append( {'object_type': 'warning', 'text' : "Cannot move '%s' to a subdirectory of itself!" % \ relative_path }) status = returnvalues.CLIENT_ERROR continue real_target = os.path.join(real_target, os.path.basename(real_path)) try: shutil.move(real_path, real_target) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) algo_list = accepted['hash_algo'] max_chunks = int(accepted['max_chunks'][-1]) pattern_list = accepted['path'] dst = accepted['dst'][-1] current_dir = accepted['current_dir'][-1].lstrip(os.sep) # All paths are relative to current_dir pattern_list = [os.path.join(current_dir, i) for i in pattern_list] if dst: dst = os.path.join(current_dir, dst) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dir = os.path.abspath( os.path.join(base_dir, current_dir.lstrip(os.sep))) if not valid_user_path(configuration, abs_dir, base_dir, True): output_objects.append({ 'object_type': 'error_text', 'text': "You're not allowed to work in %s!" % current_dir }) logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dir, current_dir)) return (output_objects, returnvalues.CLIENT_ERROR) if verbose(flags): output_objects.append({ 'object_type': 'text', 'text': "working in %s" % current_dir }) if dst: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # NOTE: dst already incorporates current_dir prefix here # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) logger.info('chksum in %s' % abs_dest) # Don't use abs_path in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % dst }) logger.warning('%s tried to %s restricted path %s !(%s)' % (client_id, op_name, abs_dest, dst)) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_dest)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot checksum to "%s": inside a read-only location!' % relative_dest }) return (output_objects, returnvalues.CLIENT_ERROR) all_lines = [] for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') output_lines = [] for hash_algo in algo_list: try: chksum_helper = _algo_map.get(hash_algo, _algo_map["md5"]) checksum = chksum_helper(abs_path, max_chunks=max_chunks) line = "%s %s\n" % (checksum, relative_path) logger.info("%s %s of %s: %s" % (op_name, hash_algo, abs_path, checksum)) output_lines.append(line) except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = {'object_type': 'file_output', 'lines': output_lines} output_objects.append(entry) all_lines += output_lines
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] status = returnvalues.OK (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] dst = accepted['dst'][-1].lstrip(os.sep) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) if dst: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) dst_mode = "wb" # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) relative_dst = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): logger.warning('%s tried to %s into restricted path %s ! (%s)' % (client_id, op_name, abs_dest, dst)) output_objects.append({ 'object_type': 'error_text', 'text': "invalid destination: '%s'" % dst }) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'accessed', [relative_path]) fd = open(abs_path, 'r') # use file directly as iterator for efficiency for line in fd: output_lines.append(line) fd.close() except Exception, exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'accessed', [relative_path], failed=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if dst: try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'modified', [dst]) out_fd = open(abs_dest, dst_mode) out_fd.writelines(output_lines) out_fd.close() logger.info('%s %s %s done' % (op_name, abs_path, abs_dest)) except Exception, exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'modified', [dst], error=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "write failed: '%s'" % exc }) logger.error("%s: write failed on '%s': %s" % (op_name, abs_dest, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({ 'object_type': 'text', 'text': "wrote %s to %s" % (relative_path, relative_dst) }) # Prevent truncate after first write dst_mode = "ab+" else: entry = { 'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines'] } if verbose(flags): entry['path'] = relative_path output_objects.append(entry) # TODO: rip this hack out into real download handler? # Force download of files when output_format == 'file_format' # This will only work for the first file matching a glob when # using file_format. # And it is supposed to only work for one file. if user_arguments_dict.has_key('output_format'): output_format = user_arguments_dict['output_format'][0] if output_format == 'file': output_objects.append({ 'object_type': 'start', 'headers': [('Content-Disposition', 'attachment; filename="%s";' % os.path.basename(abs_path))] })
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') output_lines = [] try: filedes = open(abs_path, 'r') i = 0 for line in filedes: if i >= lines: break output_lines.append(line) i += 1 filedes.close() except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = { 'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines'] } if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) # Show all if no type flags given if not byte_count(flags) and not line_count(flags)\ and not word_count(flags): flags += 'blw' for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND filewcs = [] for real_path in match: relative_path = real_path.replace(base_dir, '') (bytes, words, lines) = (0, 0, 0) try: obj = {'object_type': 'filewc', 'name': relative_path} if os.path.isdir(real_path): obj = { 'object_type': 'filewc', 'name': '%s: %s: Is a directory' % (op_name, relative_path), 'lines': 0, 'words': 0, 'bytes': 0, } filewcs.append(obj) continue fd = open(real_path, 'r') lines = 0 # use file directly as iterator for efficiency for line in fd: lines += 1 bytes += len(line) words += len(line.split()) if line_count(flags): obj['lines'] = lines if word_count(flags): obj['words'] = words if byte_count(flags): obj['bytes'] = bytes filewcs.append(obj) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'filewcs', 'filewcs' : filewcs})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) # Show all if no type flags given if not byte_count(flags) and not line_count(flags)\ and not word_count(flags): flags += 'blw' for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND filewcs = [] for abs_path in match: relative_path = abs_path.replace(base_dir, '') (bytes, words, lines) = (0, 0, 0) try: obj = {'object_type': 'filewc', 'name': relative_path} if os.path.isdir(abs_path): obj = { 'object_type': 'filewc', 'name': '%s: %s: Is a directory' % (op_name, relative_path), 'lines': 0, 'words': 0, 'bytes': 0, } filewcs.append(obj) continue fd = open(abs_path, 'r') lines = 0 # use file directly as iterator for efficiency for line in fd: lines += 1 bytes += len(line) words += len(line.split()) if line_count(flags): obj['lines'] = lines if word_count(flags): obj['words'] = words if byte_count(flags): obj['bytes'] = bytes filewcs.append(obj) except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'filewcs', 'filewcs': filewcs})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] # IMPORTANT: the CGI front end forces the input extraction to be delayed # We must manually extract and parse input here to avoid memory explosion # for huge files! # TODO: explosions still happen sometimes! # Most likely because of Apache SSL renegotiations which have # no other way of storing input extract_input = user_arguments_dict.get('__DELAYED_INPUT__', dict) logger.info('Extracting input in %s' % op_name) form = extract_input() logger.info('After extracting input in %s' % op_name) file_item = None file_name = '' user_arguments_dict = {} if form.has_key('fileupload'): file_item = form['fileupload'] file_name = file_item.filename user_arguments_dict['fileupload'] = ['true'] user_arguments_dict['path'] = [file_name] if form.has_key('path'): user_arguments_dict['path'] = [form['path'].value] if form.has_key('restrict'): user_arguments_dict['restrict'] = [form['restrict'].value] else: user_arguments_dict['restrict'] = defaults['restrict'] logger.info('Filtered input is: %s' % user_arguments_dict) # Now validate parts as usual (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) path = accepted['path'][-1] restrict = accepted['restrict'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_griddk: output_objects.append({ 'object_type': 'text', 'text': '''Grid.dk features are disabled on this site. Please contact the site admins %s if you think they should be enabled. ''' % configuration.admin_email }) return (output_objects, returnvalues.OK) logger.info('Filtered input validated with result: %s' % accepted) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) output_objects.append({'object_type': 'header', 'text': 'Uploading file'}) # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing consistent # error messages real_path = os.path.realpath(os.path.join(base_dir, path)) # Implicit destination if os.path.isdir(real_path): real_path = os.path.join(real_path, os.path.basename(file_name)) if not valid_user_path(configuration, real_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, path)) output_objects.append({ 'object_type': 'error_text', 'text': "Invalid destination (%s expands to an illegal path)" % path }) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(os.path.dirname(real_path)): output_objects.append({ 'object_type': 'error_text', 'text': "cannot write: no such file or directory: %s)" % path }) return (output_objects, returnvalues.CLIENT_ERROR) # We fork off here and redirect the user to a progress page for user # friendly output and to avoid cgi timeouts from killing the upload. # We use something like the Active State python recipe for daemonizing # to properly detach from the CGI process and continue in the background. # Please note that we only close stdio file descriptors to avoid closing # the fileupload. file_item.file.seek(0, 2) total_size = file_item.file.tell() file_item.file.seek(0, 0) try: pid = os.fork() if pid == 0: os.setsid() pid = os.fork() if pid == 0: os.chdir('/') os.umask(0) for fno in range(3): try: os.close(fno) except OSError: pass else: os._exit(0) except OSError, ose: output_objects.append({ 'object_type': 'error_text', 'text': '%s upload could not background! (%s)' % (path, str(ose).replace(base_dir, '')) }) return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = accepted['flags'] patterns = accepted['job_id'] if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep mrsl_keywords_dict = get_keywords_dict(configuration) if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in patterns: # Add file extension pattern += '.mRSL' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') try: mrsl_dict = unpickle(abs_path, logger) if not mrsl_dict: raise Exception('could not load job mRSL') for (key, val) in mrsl_dict.items(): if not key in mrsl_keywords_dict.keys(): continue if not val: continue output_lines.append('::%s::\n' % key) if 'multiplestrings' == mrsl_keywords_dict[key]['Type']: for line in val: output_lines.append('%s\n' % line) elif 'multiplekeyvalues' == mrsl_keywords_dict[key][ 'Type']: for (left, right) in val: output_lines.append('%s=%s\n' % (left, right)) else: output_lines.append('%s\n' % val) output_lines.append('\n') except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if verbose(flags): output_objects.append({ 'object_type': 'file_output', 'path': relative_path, 'lines': output_lines }) else: output_objects.append({ 'object_type': 'file_output', 'lines': output_lines })
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND submitstatuslist = [] for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') submitstatus = { 'object_type': 'submitstatus', 'name': relative_path } try: (job_status, newmsg, job_id) = new_job(abs_path, client_id, configuration, False, True) except Exception, exc: logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) job_status = False newmsg = "%s failed on '%s' (is it a valid mRSL file?)"\ % (op_name, relative_path) job_id = None if not job_status: submitstatus['status'] = False submitstatus['message'] = newmsg status = returnvalues.CLIENT_ERROR else: submitstatus['status'] = True submitstatus['job_id'] = job_id submitstatuslist.append(submitstatus) output_objects.append({ 'object_type': 'submitstatuslist', 'submitstatuslist': submitstatuslist })
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] # IMPORTANT: the CGI front end forces the input extraction to be delayed # We must manually extract and parse input here to avoid memory explosion # for huge files! # TODO: explosions still happen sometimes! # Most likely because of Apache SSL renegotiations which have # no other way of storing input extract_input = user_arguments_dict["__DELAYED_INPUT__"] logger.info("Extracting input in %s" % op_name) form = extract_input() logger.info("After extracting input in %s" % op_name) file_item = None file_name = "" user_arguments_dict = {} if form.has_key("fileupload"): file_item = form["fileupload"] file_name = file_item.filename user_arguments_dict["fileupload"] = ["true"] user_arguments_dict["path"] = [file_name] if form.has_key("path"): user_arguments_dict["path"] = [form["path"].value] if form.has_key("restrict"): user_arguments_dict["restrict"] = [form["restrict"].value] else: user_arguments_dict["restrict"] = defaults["restrict"] logger.info("Filtered input is: %s" % user_arguments_dict) # Now validate parts as usual (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler("POST"): output_objects.append( {"object_type": "error_text", "text": "Only accepting POST requests to prevent unintended updates"} ) return (output_objects, returnvalues.CLIENT_ERROR) flags = "".join(accepted["flags"]) path = accepted["path"][-1] restrict = accepted["restrict"][-1] if not configuration.site_enable_griddk: output_objects.append( { "object_type": "text", "text": """Grid.dk features are disabled on this site. Please contact the Grid admins %s if you think they should be enabled. """ % configuration.admin_email, } ) return (output_objects, returnvalues.OK) logger.info("Filtered input validated with result: %s" % accepted) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({"object_type": "text", "text": "%s using flag: %s" % (op_name, flag)}) output_objects.append({"object_type": "header", "text": "Uploading file"}) # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing consistent # error messages real_path = os.path.realpath(os.path.join(base_dir, path)) # Implicit destination if os.path.isdir(real_path): real_path = os.path.join(real_path, os.path.basename(file_name)) if not valid_user_path(real_path, base_dir, True): logger.warning("%s tried to %s restricted path %s ! (%s)" % (client_id, op_name, real_path, path)) output_objects.append( {"object_type": "error_text", "text": "Invalid destination (%s expands to an illegal path)" % path} ) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(os.path.dirname(real_path)): output_objects.append( {"object_type": "error_text", "text": "cannot write: no such file or directory: %s)" % path} ) return (output_objects, returnvalues.CLIENT_ERROR) # We fork off here and redirect the user to a progress page for user # friendly output and to avoid cgi timeouts from killing the upload. # We use something like the Active State python recipe for daemonizing # to properly detach from the CGI process and continue in the background. # Please note that we only close stdio file descriptors to avoid closing # the fileupload. file_item.file.seek(0, 2) total_size = file_item.file.tell() file_item.file.seek(0, 0) try: pid = os.fork() if pid == 0: os.setsid() pid = os.fork() if pid == 0: os.chdir("/") os.umask(0) for fno in range(3): try: os.close(fno) except OSError: pass else: os._exit(0) except OSError, ose: output_objects.append( { "object_type": "error_text", "text": "%s upload could not background! (%s)" % (path, str(ose).replace(base_dir, "")), } ) return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') output_lines = [] try: filedes = open(real_path, 'r') i = 0 for line in filedes: if i >= lines: break output_lines.append(line) i += 1 filedes.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = {'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines']} if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = accepted['flags'] patterns = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND stats = [] for real_path in match: relative_path = real_path.replace(base_dir, '') try: (stat_status, stat) = stat_path(real_path, logger) if stat_status: if verbose(flags): stat['name'] = relative_path stat['object_type'] = 'stat' stats.append(stat) else: output_objects.append({'object_type': 'error_text', 'text': stat}) status = returnvalues.SYSTEM_ERROR except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'stats', 'stats' : stats})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) defaults = signature()[1] title_entry = find_entry(output_objects, 'title') label = "%s" % configuration.site_vgrid_label title_entry['text'] = '%s Workflows' % label # NOTE: Delay header entry here to include vgrid_name (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) vgrid_name = accepted['vgrid_name'][-1] operation = accepted['operation'][-1] flags = ''.join(accepted['flags'][-1]) if not vgrid_is_owner_or_member(vgrid_name, client_id, configuration): output_objects.append({ 'object_type': 'error_text', 'text': '''You must be an owner or member of %s vgrid to access the workflows.''' % vgrid_name }) return (output_objects, returnvalues.CLIENT_ERROR) if not operation in allowed_operations: output_objects.append({ 'object_type': 'error_text', 'text': '''Operation must be one of %s.''' % ', '.join(allowed_operations) }) return (output_objects, returnvalues.OK) if operation in show_operations: # jquery support for tablesorter (and unused confirmation dialog) # table initially sorted by 0 (last update / date) refresh_call = 'ajax_workflowjobs("%s", "%s")' % (vgrid_name, flags) table_spec = { 'table_id': 'workflowstable', 'sort_order': '[[0,1]]', 'refresh_call': refresh_call } (add_import, add_init, add_ready) = man_base_js(configuration, [table_spec]) if operation == "show": add_ready += '%s;' % refresh_call add_ready += ''' /* Init variables helper as foldable but closed and with individual heights */ $(".variables-accordion").accordion({ collapsible: true, active: false, heightStyle: "content" }); /* fix and reduce accordion spacing */ $(".ui-accordion-header").css("padding-top", 0) .css("padding-bottom", 0).css("margin", 0); /* NOTE: requires managers CSS fix for proper tab bar height */ $(".workflow-tabs").tabs(); $("#logarea").scrollTop($("#logarea")[0].scrollHeight); ''' title_entry['script']['advanced'] += add_import title_entry['script']['init'] += add_init title_entry['script']['ready'] += add_ready output_objects.append({ 'object_type': 'html_form', 'text': man_base_html(configuration) }) output_objects.append({ 'object_type': 'header', 'text': '%s Workflows for %s' % (label, vgrid_name) }) logger.info('vgridworkflows %s %s' % (vgrid_name, operation)) # Iterate through jobs and list details for each trigger_jobs = [] log_content = '' if operation in list_operations: trigger_job_dir = os.path.join( configuration.vgrid_home, os.path.join(vgrid_name, '.%s.jobs' % configuration.vgrid_triggers)) trigger_job_pending_dir = os.path.join(trigger_job_dir, 'pending_states') trigger_job_final_dir = os.path.join(trigger_job_dir, 'final_states') if makedirs_rec(trigger_job_pending_dir, configuration) \ and makedirs_rec(trigger_job_final_dir, configuration): abs_vgrid_dir = '%s/' \ % os.path.abspath(os.path.join(configuration.vgrid_files_home, vgrid_name)) for filename in os.listdir(trigger_job_pending_dir): trigger_job_filepath = \ os.path.join(trigger_job_pending_dir, filename) trigger_job = unpickle(trigger_job_filepath, logger) serverjob_filepath = \ os.path.join(configuration.mrsl_files_dir, os.path.join( client_id_dir(trigger_job['owner']), '%s.mRSL' % trigger_job['jobid'])) serverjob = unpickle(serverjob_filepath, logger) if serverjob: if serverjob['STATUS'] in pending_states: trigger_event = trigger_job['event'] trigger_rule = trigger_job['rule'] trigger_action = trigger_event['event_type'] trigger_time = time.ctime(trigger_event['time_stamp']) trigger_path = '%s %s' % \ (trigger_event['src_path'].replace( abs_vgrid_dir, ''), trigger_event['dest_path'].replace( abs_vgrid_dir, '')) job = { 'object_type': 'trigger_job', 'job_id': trigger_job['jobid'], 'rule_id': trigger_rule['rule_id'], 'path': trigger_path, 'action': trigger_action, 'time': trigger_time, 'status': serverjob['STATUS'] } if not job['rule_id'].startswith(img_trigger_prefix) \ or verbose(flags): trigger_jobs.append(job) elif serverjob['STATUS'] in final_states: src_path = os.path.join(trigger_job_pending_dir, filename) dest_path = os.path.join(trigger_job_final_dir, filename) move_file(src_path, dest_path, configuration) else: logger.error( 'Trigger job: %s, unknown state: %s' % (trigger_job['jobid'], serverjob['STATUS'])) log_content = read_trigger_log(configuration, vgrid_name, flags) if operation in show_operations: # Always run as rule creator to avoid users being able to act on behalf # of ANY other user using triggers (=exploit) extra_fields = [ ('path', None), ('match_dirs', ['False', 'True']), ('match_recursive', ['False', 'True']), ('changes', [keyword_all] + valid_trigger_changes), ('action', [keyword_auto] + valid_trigger_actions), ('arguments', None), ('run_as', client_id), ] # NOTE: we do NOT show saved template contents - see addvgridtriggers optional_fields = [('rate_limit', None), ('settle_time', None)] # Only include system triggers in verbose mode if verbose(flags): system_filter = [] else: system_filter = [('rule_id', '%s_.*' % img_trigger_prefix)] (init_status, oobjs) = vgrid_add_remove_table(client_id, vgrid_name, 'trigger', 'vgridtrigger', configuration, extra_fields + optional_fields, filter_items=system_filter) if not init_status: output_objects.append({ 'object_type': 'error_text', 'text': 'failed to load triggers: %s' % oobjs }) return (output_objects, returnvalues.SYSTEM_ERROR) # Generate variable helper values for a few concrete samples for help # text vars_html = '' dummy_rule = {'run_as': client_id, 'vgrid_name': vgrid_name} samples = [('input.txt', 'modified'), ('input/image42.raw', 'changed')] for (path, change) in samples: vgrid_path = os.path.join(vgrid_name, path) vars_html += "<b>Expanded variables when %s is %s:</b><br/>" % \ (vgrid_path, change) expanded = get_path_expand_map(vgrid_path, dummy_rule, change) for (key, val) in expanded.items(): vars_html += " %s: %s<br/>" % (key, val) commands_html = '' commands = get_usage_map(configuration) for usage in commands.values(): commands_html += " %s<br/>" % usage helper_html = """ <div class='variables-accordion'> <h4>Help on available trigger variable names and values</h4> <p> Triggers can use a number of helper variables on the form +TRIGGERXYZ+ to dynamically act on targets. Some of the values are bound to the rule owner the %s while the remaining ones are automatically expanded for the particular trigger target as shown in the following examples:<br/> %s </p> <h4>Help on available trigger commands and arguments</h4> <p> It is possible to set up trigger rules that basically run any operation with a side effect you could manually do on %s. I.e. like submitting/cancelling a job, creating/moving/deleting a file or directory and so on. When you select 'command' as the action for a trigger rule, you have the following commands at your disposal:<br/> %s </p> </div> """ % (label, vars_html, configuration.short_title, commands_html) # Make page with manage triggers tab and active jobs and log tab output_objects.append({ 'object_type': 'html_form', 'text': ''' <div id="wrap-tabs" class="workflow-tabs"> <ul> <li><a href="#manage-tab">Manage Triggers</a></li> <li><a href="#jobs-tab">Active Trigger Jobs</a></li> </ul> ''' }) # Display existing triggers and form to add new ones output_objects.append({ 'object_type': 'html_form', 'text': ''' <div id="manage-tab"> ''' }) output_objects.append({ 'object_type': 'sectionheader', 'text': 'Manage Triggers' }) output_objects.extend(oobjs) output_objects.append({ 'object_type': 'html_form', 'text': helper_html }) if configuration.site_enable_crontab: output_objects.append({ 'object_type': 'html_form', 'text': ''' <p>You can combine these workflows with the personal ''' }) output_objects.append({ 'object_type': 'link', 'destination': 'crontab.py', 'class': 'crontablink iconspace', 'text': 'schedule task' }) output_objects.append({ 'object_type': 'html_form', 'text': ''' facilities in case you want to trigger flows at given times rather than only in reaction to file system events.</p> ''' }) output_objects.append({ 'object_type': 'html_form', 'text': ''' </div> ''' }) # Display active trigger jobs and recent logs for this vgrid output_objects.append({ 'object_type': 'html_form', 'text': ''' <div id="jobs-tab"> ''' }) output_objects.append({ 'object_type': 'sectionheader', 'text': 'Active Trigger Jobs' }) output_objects.append({ 'object_type': 'table_pager', 'entry_name': 'job', 'default_entries': default_pager_entries }) output_objects.append({ 'object_type': 'trigger_job_list', 'trigger_jobs': trigger_jobs }) if operation in show_operations: output_objects.append({ 'object_type': 'sectionheader', 'text': 'Trigger Log' }) output_objects.append({ 'object_type': 'trigger_log', 'log_content': log_content }) if operation in show_operations: output_objects.append({ 'object_type': 'html_form', 'text': ''' </div> ''' }) output_objects.append({ 'object_type': 'html_form', 'text': ''' </div> ''' }) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: logger.error("jobstatus input validation failed: %s" % accepted) return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) max_jobs = int(accepted['max_jobs'][-1]) order = 'unsorted ' if sorted(flags): order = 'sorted ' patterns = accepted['job_id'] project_names = accepted['project_name'] if len(project_names) > 0: for project_name in project_names: project_name_job_ids = \ get_job_ids_with_specified_project_name(client_id, project_name, configuration.mrsl_files_dir, logger) patterns.extend(project_name_job_ids) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep output_objects.append({'object_type': 'header', 'text' : '%s %s job status' % \ (configuration.short_title, order)}) if not patterns: output_objects.append({ 'object_type': 'error_text', 'text': 'No job_id specified!' }) return (output_objects, returnvalues.NO_SUCH_JOB_ID) if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) if not os.path.isdir(base_dir): output_objects.append( {'object_type': 'error_text', 'text' : ('You have not been created as a user on the %s server! ' \ 'Please contact the %s team.') % \ (configuration.short_title, configuration.short_title)}) return (output_objects, returnvalues.CLIENT_ERROR) filelist = [] for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to # avoid leaking information about file system layout while # allowing consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match.... if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % pattern }) status = returnvalues.CLIENT_ERROR else: filelist += match if sorted(flags): sort(filelist) if max_jobs > 0 and max_jobs < len(filelist): output_objects.append({ 'object_type': 'text', 'text': 'Only showing first %d of the %d matching jobs as requested' % (max_jobs, len(filelist)) }) filelist = filelist[:max_jobs] # Iterate through jobs and list details for each job_list = {'object_type': 'job_list', 'jobs': []} for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') job_dict = unpickle(filepath, logger) if not job_dict: status = returnvalues.CLIENT_ERROR output_objects.append( {'object_type': 'error_text', 'text' : 'No such job: %s (could not load mRSL file %s)' % \ (job_id, filepath)}) continue # Expand any job variables before use job_dict = expand_variables(job_dict) job_obj = {'object_type': 'job', 'job_id': job_id} job_obj['status'] = job_dict['STATUS'] time_fields = [ 'VERIFIED', 'VERIFIED_TIMESTAMP', 'RECEIVED_TIMESTAMP', 'QUEUED_TIMESTAMP', 'SCHEDULE_TIMESTAMP', 'EXECUTING_TIMESTAMP', 'FINISHED_TIMESTAMP', 'FAILED_TIMESTAMP', 'CANCELED_TIMESTAMP', ] for name in time_fields: if job_dict.has_key(name): # time objects cannot be marshalled, asctime if timestamp try: job_obj[name.lower()] = time.asctime(job_dict[name]) except Exception, exc: # not a time object, just add job_obj[name.lower()] = job_dict[name] ########################################### # ARC job status retrieval on demand: # But we should _not_ update the status in the mRSL files, since # other MiG code might rely on finding only valid "MiG" states. if configuration.arc_clusters and \ job_dict.get('UNIQUE_RESOURCE_NAME', 'unset') == 'ARC' \ and job_dict['STATUS'] == 'EXECUTING': try: home = os.path.join(configuration.user_home, client_dir) arcsession = arc.Ui(home) arcstatus = arcsession.jobStatus(job_dict['EXE']) job_obj['status'] = arcstatus['status'] except arc.ARCWrapperError, err: logger.error('Error retrieving ARC job status: %s' % \ err.what()) job_obj['status'] += '(Error: ' + err.what() + ')' except arc.NoProxyError, err: logger.error('While retrieving ARC job status: %s' % \ err.what()) job_obj['status'] += '(Error: ' + err.what() + ')'
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') output_lines = [] # We search for the last 'lines' lines by beginning from the end of # the file and exponetially backtracking until the backtrack # contains at least 'lines' lines or we're back to the beginning of # the file. # At that point we skip any extra lines before printing. try: filedes = open(abs_path, 'r') # Go to end of file and backtrack backstep = 1 newlines = 0 filedes.seek(0, 2) length = filedes.tell() while backstep < length and newlines <= lines: backstep *= 2 if backstep > length: backstep = length filedes.seek(-backstep, 2) newlines = len(filedes.readlines()) if length: # Go back after reading to end of file filedes.seek(-backstep, 2) # Skip any extra lines caused by the exponential backtracking. # We could probably speed up convergence with binary search... while newlines > lines: dummy = filedes.readline() newlines -= 1 backstep = length - filedes.tell() # Now we're at the wanted spot - print rest of file for _ in range(newlines): output_lines.append(filedes.readline()) filedes.close() except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = { 'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines'] } if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
arcstatus = arcsession.jobStatus(job_dict['EXE']) job_obj['status'] = arcstatus['status'] except arc.ARCWrapperError, err: logger.error('Error retrieving ARC job status: %s' % \ err.what()) job_obj['status'] += '(Error: ' + err.what() + ')' except arc.NoProxyError, err: logger.error('While retrieving ARC job status: %s' % \ err.what()) job_obj['status'] += '(Error: ' + err.what() + ')' except Exception, err: logger.error('Error retrieving ARC job status: %s' % err) job_obj['status'] += '(Error during retrieval)' exec_histories = [] if verbose(flags): if job_dict.has_key('EXECUTE'): command_line = '; '.join(job_dict['EXECUTE']) if len(command_line) > 256: job_obj['execute'] = '%s ...' % command_line[:252] else: job_obj['execute'] = command_line res_conf = job_dict.get('RESOURCE_CONFIG', {}) if res_conf.has_key('RESOURCE_ID'): public_id = res_conf['RESOURCE_ID'] if res_conf.get('ANONYMOUS', True): public_id = anon_resource_id(public_id) job_obj['resource'] = public_id if job_dict.get('PUBLICNAME', False): job_obj['resource'] += ' (alias %(PUBLICNAME)s)' % job_dict if job_dict.has_key('RESOURCE_VGRID'):