def is_owner( client_id, unique_config_name, config_home, logger, ): """Check that client_id is listed in pickled owners file""" config_path = os.path.abspath( os.path.join(config_home, unique_config_name, 'owners')) # Check validity of unique_config_name # Automatic configuration extraction configuration = None if not valid_user_path(configuration, config_path, config_home): # Extract caller information from traceback import format_stack caller = ''.join(format_stack()[:-1]).strip() logger.warning( """is_owner caught possible illegal directory traversal attempt by client: '%s' unique name: '%s' caller: %s""" % (client_id, unique_config_name, caller)) return False return is_item_in_pickled_list(config_path, client_id, logger)
def is_owner( client_id, unique_config_name, config_home, logger, ): """Check that client_id is listed in pickled owners file""" config_path = os.path.abspath(os.path.join(config_home, unique_config_name, 'owners')) # Check validity of unique_config_name if not valid_user_path(config_path, config_home): # Extract caller information from traceback import format_stack caller = ''.join(format_stack()[:-1]).strip() logger.warning("""is_owner caught possible illegal directory traversal attempt by client: '%s' unique name: '%s' caller: %s""" % (client_id, unique_config_name, caller)) return False return is_item_in_pickled_list(config_path, client_id, logger)\ or is_item_in_pickled_list(config_path, old_id_format(client_id), logger)
def _parse_form_xfer(xfer, user_args, client_id, configuration): """Parse xfer request (i.e. copy, move or upload) file/dir entries from user_args. """ _logger = configuration.logger files, rejected = [], [] i = 0 client_dir = client_id_dir(client_id) base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep xfer_pattern = 'freeze_%s_%%d' % xfer for i in xrange(max_freeze_files): if user_args.has_key(xfer_pattern % i): source_path = user_args[xfer_pattern % i][-1].strip() source_path = os.path.normpath(source_path).lstrip(os.sep) _logger.debug('found %s entry: %s' % (xfer, source_path)) if not source_path: continue try: valid_path(source_path) except Exception, exc: rejected.append('invalid path: %s (%s)' % (source_path, exc)) continue # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath( os.path.join(base_dir, source_path)) # Prevent out-of-bounds, and restrict some greedy targets if not valid_user_path(configuration, abs_path, base_dir, True): _logger.error('found illegal directory traversal %s entry: %s' % (xfer, source_path)) rejected.append('invalid path: %s (%s)' % (source_path, 'illegal path!')) continue elif os.path.exists(abs_path) and os.path.samefile(abs_path, base_dir): _logger.warning('refusing archival of entire user home %s: %s' % (xfer, source_path)) rejected.append('invalid path: %s (%s)' % (source_path, 'entire home not allowed!')) continue elif in_vgrid_share(configuration, abs_path) == source_path: _logger.warning( 'refusing archival of entire vgrid shared folder %s: %s' % (xfer, source_path)) rejected.append('invalid path: %s (%s)' % (source_path, 'entire %s share not allowed!' % configuration.site_vgrid_label)) continue # expand any dirs recursively if os.path.isdir(abs_path): for (root, dirnames, filenames) in os.walk(abs_path): for subname in filenames: abs_sub = os.path.join(root, subname) sub_base = root.replace(abs_path, source_path) sub_path = os.path.join(sub_base, subname) files.append((abs_sub, sub_path.lstrip(os.sep))) else: files.append((abs_path, source_path.lstrip(os.sep)))
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) vgrid_name = accepted['vgrid_name'][-1] path = accepted['path'][-1] if not vgrid_is_owner_or_member(vgrid_name, client_id, configuration): output_objects.append({'object_type': 'error_text', 'text': '''You must be an owner or member of %s %s to access the private files.''' % (vgrid_name, configuration.site_vgrid_label)}) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.vgrid_private_base, vgrid_name)) + os.sep # Strip leading slashes to avoid join() throwing away prefix rel_path = path.lstrip(os.sep) real_path = os.path.abspath(os.path.join(base_dir, rel_path)) if not valid_user_path(real_path, base_dir, True): output_objects.append({'object_type': 'error_text', 'text': '''You are not allowed to use paths outside %s private files dir.''' % configuration.site_vgrid_label}) return (output_objects, returnvalues.CLIENT_ERROR) try: private_fd = open(real_path, 'rb') entry = {'object_type': 'binary', 'data': private_fd.read()} # Cut away all the usual web page formatting to show only contents output_objects = [{'object_type': 'start', 'headers': []}, entry, {'object_type': 'script_status'}, {'object_type': 'end'}] private_fd.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text' : 'Error reading %s private file (%s)' % (configuration.site_vgrid_label, exc)}) return (output_objects, returnvalues.SYSTEM_ERROR)
def parse_form_upload(user_args, client_id, configuration, base_dir): """Parse upload file and chunk entries from user_args. Chunk limits are extracted from content-range http header in environment. Files are considered to be inside uplad tmp dir inside base_dir. """ files, rejected = [], [] logger = configuration.logger cache_dir = os.path.join(base_dir, upload_tmp_dir) + os.sep # TODO: we only support single filename and chunk for now; extend? #for name_index in xrange(max_upload_files): # if user_args.has_key(filename_field) and \ # len(user_args[filename_field]) > name_index: for name_index in [0]: if user_args.has_key(filename_field): if isinstance(user_args[filename_field], basestring): filename = user_args[filename_field] else: filename = user_args[filename_field][name_index] logger.info('found name: %s' % filename) else: # No more files break if not filename.strip(): continue try: filename = strip_dir(filename) valid_path(filename) except Exception, exc: logger.error('invalid filename: %s' % filename) rejected.append((filename, 'invalid filename: %s (%s)' % \ (filename, exc))) continue rel_path = os.path.join(upload_tmp_dir, filename) real_path = os.path.abspath(os.path.join(base_dir, rel_path)) if not valid_user_path(real_path, cache_dir, True): logger.error('%s tried to access restricted path %s ! (%s)' % (client_id, real_path, cache_dir)) rejected.append("Invalid path (%s expands to an illegal path)" \ % filename) continue #for chunk_index in xrange(max_upload_chunks): # if user_args.has_key(files_field) and \ # len(user_args[files_field]) > chunk_index: for chunk_index in [0]: if user_args.has_key(files_field): chunk = user_args[files_field][chunk_index] else: break configuration.logger.debug('find chunk range: %s' % filename) (chunk_first, chunk_last) = extract_chunk_region(configuration) if len(chunk) > upload_block_size: configuration.logger.error('skip bigger than allowed chunk') continue elif chunk_last < 0: chunk_last = len(chunk) - 1 files.append((rel_path, (chunk, chunk_first, chunk_last)))
def get_fs_path(configuration, abs_path, root, chroot_exceptions): """Internal helper to translate path with chroot and invisible files in mind. Also assures general path character restrictions are applied. Automatically expands to abs path to avoid traversal issues with e.g. MYVGRID/../bla that would expand to vgrid_files_home/bla instead of bla in user home if left as is. """ try: valid_path(abs_path) except: raise ValueError("Invalid path characters") if not valid_user_path(configuration, abs_path, root, True, chroot_exceptions): raise ValueError("Illegal path access attempt") return abs_path
def _parse_form_xfer(xfer, user_args, client_id, configuration): """Parse xfer request (i.e. copy, move or upload) file/dir entries from user_args. """ files, rejected = [], [] i = 0 client_dir = client_id_dir(client_id) base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep xfer_pattern = 'freeze_%s_%%d' % xfer for i in xrange(max_freeze_files): if user_args.has_key(xfer_pattern % i): source_path = user_args[xfer_pattern % i][-1].strip() configuration.logger.debug('found %s entry: %s' % (xfer, source_path)) if not source_path: continue try: valid_path(source_path) except Exception, exc: rejected.append('invalid path: %s (%s)' % (source_path, exc)) continue source_path = os.path.normpath(source_path).lstrip(os.sep) real_path = os.path.abspath(os.path.join(base_dir, source_path)) if not valid_user_path(real_path, base_dir, True): rejected.append('invalid path: %s (%s)' % \ (source_path, 'illegal path!')) continue # expand any dirs recursively if os.path.isdir(real_path): for (root, dirnames, filenames) in os.walk(real_path): for subname in filenames: real_sub = os.path.join(root, subname) sub_base = root.replace(real_path, source_path) sub_path = os.path.join(sub_base, subname) files.append((real_sub, sub_path)) else: files.append((real_path, source_path))
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] name_pattern = accepted['name'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) dir_listings = [] output_objects.append({ 'object_type': 'dir_listings', 'dir_listings': dir_listings, 'flags': flags, }) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: output_lines = [] relative_path = real_path.replace(base_dir, '') entries = [] dir_listing = { 'object_type': 'dir_listing', 'relative_path': relative_path, 'entries': entries, 'flags': flags, } dir_listings.append(dir_listing) try: for (root, dirs, files) in os.walk(real_path): for filename in fnmatch.filter(files, name_pattern): real_path = os.path.join(root, filename) relative_path = real_path.replace(base_dir, '') if not valid_user_path(real_path, base_dir, True): continue file_obj = { 'object_type': 'direntry', 'type': 'file', 'name': filename, 'file_with_dir': relative_path, 'flags': flags, 'special': '', } entries.append(file_obj) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if verbose(flags): output_objects.append({'object_type': 'file_output', 'path': relative_path, 'lines': output_lines}) else: output_objects.append({'object_type': 'file_output', 'lines': output_lines})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) patterns = accepted['job_id'] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) if not patterns: output_objects.append({ 'object_type': 'error_text', 'text': 'No job_id specified!' }) return (output_objects, returnvalues.NO_SUCH_JOB_ID) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep filelist = [] keywords_dict = mrslkeywords.get_keywords_dict(configuration) for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % pattern }) status = returnvalues.CLIENT_ERROR else: filelist += match # resubmit is hard on the server if len(filelist) > 100: output_objects.append({ 'object_type': 'error_text', 'text': 'Too many matching jobs (%s)!' % len(filelist) }) return (output_objects, returnvalues.CLIENT_ERROR) resubmitobjs = [] status = returnvalues.OK for filepath in filelist: mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') # ("Resubmitting job with job_id: %s" % job_id) resubmitobj = {'object_type': 'resubmitobj', 'job_id': job_id} mrsl_dict = unpickle(filepath, logger) if not mrsl_dict: resubmitobj['message'] = "No such job: %s (%s)" % (job_id, mrsl_file) status = returnvalues.CLIENT_ERROR resubmitobjs.append(resubmitobj) continue resubmit_items = keywords_dict.keys() # loop selected keywords and create mRSL string resubmit_job_string = '' for dict_elem in resubmit_items: value = '' # Extract job value with fallback to default to support optional # fields job_value = mrsl_dict.get(dict_elem, keywords_dict[dict_elem]['Value']) if keywords_dict[dict_elem]['Type'].startswith( 'multiplekeyvalues'): for (elem_key, elem_val) in job_value: if elem_key: value += '%s=%s\n' % (str(elem_key).strip(), str(elem_val).strip()) elif keywords_dict[dict_elem]['Type'].startswith('multiple'): for elem in job_value: if elem: value += '%s\n' % str(elem).rstrip() else: if str(job_value): value += '%s\n' % str(job_value).rstrip() # Only insert keywords with an associated value if value: if value.rstrip() != '': resubmit_job_string += '''::%s:: %s ''' % (dict_elem, value.rstrip()) # save tempfile (filehandle, tempfilename) = \ tempfile.mkstemp(dir=configuration.mig_system_files, text=True) os.write(filehandle, resubmit_job_string) os.close(filehandle) # submit job the usual way (new_job_status, msg, new_job_id) = new_job(tempfilename, client_id, configuration, False, True) if not new_job_status: resubmitobj['status'] = False resubmitobj['message'] = msg status = returnvalues.SYSTEM_ERROR resubmitobjs.append(resubmitobj) continue # o.out("Resubmit failed: %s" % msg) # o.reply_and_exit(o.ERROR) resubmitobj['status'] = True resubmitobj['new_job_id'] = new_job_id resubmitobjs.append(resubmitobj) # o.out("Resubmit successful: %s" % msg) # o.out("%s" % msg) output_objects.append({ 'object_type': 'resubmitobjs', 'resubmitobjs': resubmitobjs }) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False, op_menu=client_id) defaults = signature()[1] (validate_status, accepted) = validate_input(user_arguments_dict, defaults, output_objects, allow_rejects=False) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) queue = accepted['queue'][-1] action = accepted['action'][-1] iosessionid = accepted['iosessionid'][-1] msg = accepted['msg'][-1] msg_id = accepted['msg_id'][-1] # Web format for cert access and no header for SID access if client_id: output_objects.append({'object_type': 'header', 'text' : 'Message queue %s' % action}) else: output_objects.append({'object_type': 'start'}) # Always return at least a basic file_output entry file_entry = {'object_type': 'file_output', 'lines': [], 'wrap_binary': True, 'wrap_targets': ['lines']} if not action in valid_actions: output_objects.append({'object_type': 'error_text', 'text' : 'Invalid action "%s" (supported: %s)' % \ (action, ', '.join(valid_actions))}) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) if action in post_actions and not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) # Find user home from session or certificate if iosessionid: client_home = os.path.realpath(os.path.join(configuration.webserver_home, iosessionid)) client_dir = os.path.basename(client_home) elif client_id: client_dir = client_id_dir(client_id) else: output_objects.append({'object_type': 'error_text', 'text' : 'Either certificate or session ID is required' }) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if not os.path.isdir(base_dir): output_objects.append({'object_type': 'error_text', 'text' : 'No matching session or user home!'}) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) mqueue_base = os.path.join(base_dir, mqueue_prefix) + os.sep default_queue_dir = os.path.join(mqueue_base, default_mqueue) # Create mqueue base and default queue dir if missing if not os.path.exists(default_queue_dir): try: os.makedirs(default_queue_dir) except: pass queue_path = os.path.abspath(os.path.join(mqueue_base, queue)) if not valid_user_path(queue_path, mqueue_base): output_objects.append({'object_type': 'error_text', 'text' : 'Invalid queue name: "%s"' % queue}) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) lock_path = os.path.join(mqueue_base, lock_name) lock_handle = open(lock_path, 'a') fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) status = returnvalues.OK if action == "interactive": output_objects.append({'object_type': 'text', 'text' : ''' Fill in the fields below to control and access your personal message queues. Jobs can receive from and send to the message queues during execution, and use them as a means of job inter-communication. Expect message queue operations to take several seconds on the resources, however. That is, use it for tasks like orchestrating long running jobs, and not for low latency communication. '''}) html = ''' <form name="mqueueform" method="post" action="mqueue.py"> <table class="mqueue"> <tr><td class=centertext> </td></tr> <tr><td> Action:<br /> <input type=radio name=action value="create" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />create queue <input type=radio name=action checked value="send" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=false;" />send message to queue <input type=radio name=action value="receive" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />receive message from queue <input type=radio name=action value="remove" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />remove queue <input type=radio name=action value="listqueues" onclick="javascript: document.mqueueform.queue.disabled=true; document.mqueueform.msg.disabled=true;" />list queues <input type=radio name=action value="listmessages" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />list messages <input type=radio name=action value="show" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />show message </td></tr> <tr><td> Queue:<br /> <input type=text size=60 name=queue value="%s" /> </td></tr> <tr><td> <div id="msgfieldf"> <input type=text size=60 name=msg value="%s" /><br /> </div> </td></tr> <tr><td> <input type="submit" value="Apply" /> </td></tr> </table> </form> ''' % (queue, msg) output_objects.append({'object_type': 'html_form', 'text' : html}) output_objects.append({'object_type': 'text', 'text': ''' Further live job control is avalable through the live I/O interface. They provide a basic interface for centrally managing input and output files for active jobs. ''' }) output_objects.append({'object_type': 'link', 'destination': 'liveio.py', 'text': 'Live I/O interface'}) return (output_objects, returnvalues.OK) elif action == 'create': try: os.mkdir(queue_path) output_objects.append({'object_type': 'text', 'text': 'New "%s" queue created' % queue}) except Exception, err: output_objects.append({'object_type': 'error_text', 'text' : 'Could not create "%s" queue: "%s"' % \ (queue, err)}) status = returnvalues.CLIENT_ERROR
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) src_list = accepted['src'] dst = accepted['dst'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK real_dest = base_dir + dst dst_list = glob.glob(real_dest) if not dst_list: # New destination? if not glob.glob(os.path.dirname(real_dest)): output_objects.append({'object_type': 'error_text', 'text' : 'Illegal dst path provided!'}) return (output_objects, returnvalues.CLIENT_ERROR) else: dst_list = [real_dest] # Use last match in case of multiple matches dest = dst_list[-1] if len(dst_list) > 1: output_objects.append( {'object_type': 'warning', 'text' : 'dst (%s) matches multiple targets - using last: %s' % (dst, dest)}) real_dest = os.path.abspath(dest) # Don't use real_path in output as it may expose underlying # fs layout. relative_dest = real_dest.replace(base_dir, '') if not valid_user_path(real_dest, base_dir, True): logger.warning('%s tried to %s to restricted path %s ! (%s)' % (client_id, op_name, real_dest, dst)) output_objects.append( {'object_type': 'error_text', 'text' : "Invalid path! (%s expands to an illegal path)" % dst}) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in src_list: unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'error_text', 'text' : '%s: no such file or directory! %s' % (op_name, pattern)}) status = returnvalues.CLIENT_ERROR for real_path in match: relative_path = real_path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file', 'name' : relative_path}) if os.path.islink(real_path): output_objects.append( {'object_type': 'warning', 'text' : "You're not allowed to move entire %s shared dirs!" % configuration.site_vgrid_label}) status = returnvalues.CLIENT_ERROR continue # If destination is a directory the src should be moved in there # Move with existing directory as target replaces the directory! real_target = real_dest if os.path.isdir(real_target): if os.path.samefile(real_target, real_path): output_objects.append( {'object_type': 'warning', 'text' : "Cannot move '%s' to a subdirectory of itself!" % \ relative_path }) status = returnvalues.CLIENT_ERROR continue real_target = os.path.join(real_target, os.path.basename(real_path)) try: shutil.move(real_path, real_target) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) patterns = accepted['job_id'] if not patterns: output_objects.append({'object_type': 'error_text', 'text' : 'No job_id specified!'}) return (output_objects, returnvalues.NO_SUCH_JOB_ID) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep filelist = [] keywords_dict = mrslkeywords.get_keywords_dict(configuration) for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append( {'object_type': 'error_text', 'text' : '%s: You do not have any matching job IDs!' % pattern}) status = returnvalues.CLIENT_ERROR else: filelist += match # resubmit is hard on the server if len(filelist) > 100: output_objects.append({'object_type': 'error_text', 'text' : 'Too many matching jobs (%s)!' % len(filelist)}) return (output_objects, returnvalues.CLIENT_ERROR) resubmitobjs = [] status = returnvalues.OK for filepath in filelist: mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') # ("Resubmitting job with job_id: %s" % job_id) resubmitobj = {'object_type': 'resubmitobj', 'job_id': job_id} mrsl_dict = unpickle(filepath, logger) if not mrsl_dict: resubmitobj['message'] = "No such job: %s (%s)" % (job_id, mrsl_file) status = returnvalues.CLIENT_ERROR resubmitobjs.append(resubmitobj) continue resubmit_items = keywords_dict.keys() # loop selected keywords and create mRSL string resubmit_job_string = '' for dict_elem in resubmit_items: value = '' # Extract job value with fallback to default to support optional # fields job_value = mrsl_dict.get(dict_elem, keywords_dict[dict_elem]['Value']) if keywords_dict[dict_elem]['Type'].startswith('multiplekeyvalues'): for (elem_key, elem_val) in job_value: if elem_key: value += '%s=%s\n' % (str(elem_key).strip(), str(elem_val).strip()) elif keywords_dict[dict_elem]['Type'].startswith('multiple'): for elem in job_value: if elem: value += '%s\n' % str(elem).rstrip() else: if str(job_value): value += '%s\n' % str(job_value).rstrip() # Only insert keywords with an associated value if value: if value.rstrip() != '': resubmit_job_string += '''::%s:: %s ''' % (dict_elem, value.rstrip()) # save tempfile (filehandle, tempfilename) = \ tempfile.mkstemp(dir=configuration.mig_system_files, text=True) os.write(filehandle, resubmit_job_string) os.close(filehandle) # submit job the usual way (new_job_status, msg, new_job_id) = new_job(tempfilename, client_id, configuration, False, True) if not new_job_status: resubmitobj['status'] = False resubmitobj['message'] = msg status = returnvalues.SYSTEM_ERROR resubmitobjs.append(resubmitobj) continue # o.out("Resubmit failed: %s" % msg) # o.reply_and_exit(o.ERROR) resubmitobj['status'] = True resubmitobj['new_job_id'] = new_job_id resubmitobjs.append(resubmitobj) # o.out("Resubmit successful: %s" % msg) # o.out("%s" % msg) output_objects.append({'object_type': 'resubmitobjs', 'resubmitobjs' : resubmitobjs}) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] status = returnvalues.OK (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] dst = accepted['dst'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) if dst: dst_mode = "wb" real_dst = os.path.join(base_dir, dst) relative_dst = real_dst.replace(base_dir, '') if not valid_user_path(real_dst, base_dir, True): logger.warning('%s tried to %s into restricted path %s ! (%s)' % (client_id, op_name, real_dst, dst)) output_objects.append({'object_type': 'error_text', 'text': "invalid destination: '%s'" % \ dst}) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: output_lines = [] relative_path = real_path.replace(base_dir, '') try: fd = open(real_path, 'r') # use file directly as iterator for efficiency for line in fd: output_lines.append(line) fd.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if dst: try: out_fd = open(real_dst, dst_mode) out_fd.writelines(output_lines) out_fd.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "write failed: '%s'" % exc}) logger.error("%s: write failed on '%s': %s" % (op_name, real_dst, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'text', 'text': "wrote %s to %s" % (relative_path, relative_dst)}) # Prevent truncate after first write dst_mode = "ab+" else: entry = {'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines']} if verbose(flags): entry['path'] = relative_path output_objects.append(entry) # TODO: rip this hack out into real download handler? # Force download of files when output_format == 'file_format' # This will only work for the first file matching a glob when # using file_format. # And it is supposed to only work for one file. if user_arguments_dict.has_key('output_format'): output_format = user_arguments_dict['output_format'][0] if output_format == 'file': output_objects.append( {'object_type': 'start', 'headers': [('Content-Disposition', 'attachment; filename="%s";' % \ os.path.basename(real_path))]})
def parse_form_upload(user_args, user_id, configuration, base_dir, dst_dir, reject_write=False): """Parse upload file and chunk entries from user_args. Chunk limits are extracted from content-range http header in environment. Existing files are automatically taken from upload_tmp_dir and uploads go into dst_dir inside base_dir. The optional reject_write argument is used for delayed refusal if someone tries to upload to a read-only sharelink. """ files, rejected = [], [] logger = configuration.logger rel_dst_dir = dst_dir.replace(base_dir, '') # TODO: we only support single filename and chunk for now; extend? # for name_index in xrange(max_upload_files): # if user_args.has_key(filename_field) and \ # len(user_args[filename_field]) > name_index: for name_index in [0]: if user_args.has_key(filename_field): if isinstance(user_args[filename_field], basestring): filename = user_args[filename_field] else: filename = user_args[filename_field][name_index] logger.info('found name: %s' % filename) else: # No more files break if not filename.strip(): continue if reject_write: rejected.append((filename, 'read-only share: upload refused!')) continue try: filename = strip_dir(filename) valid_path(filename) except Exception, exc: logger.error('invalid filename: %s' % filename) rejected.append( (filename, 'invalid filename: %s (%s)' % (filename, exc))) continue rel_path = os.path.join(rel_dst_dir, filename) # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(os.path.join(base_dir, rel_path)) if not valid_user_path(configuration, abs_path, dst_dir, True): logger.error('%s tried to access restricted path %s ! (%s)' % (user_id, abs_path, dst_dir)) rejected.append("Invalid path (%s expands to an illegal path)" % filename) continue # for chunk_index in xrange(max_upload_chunks): # if user_args.has_key(files_field) and \ # len(user_args[files_field]) > chunk_index: for chunk_index in [0]: if user_args.has_key(files_field): chunk = user_args[files_field][chunk_index] else: break configuration.logger.debug('find chunk range: %s' % filename) (chunk_first, chunk_last) = extract_chunk_region(configuration) if len(chunk) > upload_block_size: configuration.logger.error('skip bigger than allowed chunk') continue elif chunk_last < 0: chunk_last = len(chunk) - 1 files.append((rel_path, (chunk, chunk_first, chunk_last)))
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) path = accepted['path'][-1] chosen_newline = accepted['newline'][-1] submitjob = accepted['submitjob'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_jobs and submitjob: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep # HTML spec dictates newlines in forms to be MS style (\r\n) # rather than un*x style (\n): change if requested. form_newline = '\r\n' allowed_newline = {'unix': '\n', 'mac': '\r', 'windows': '\r\n'} output_objects.append({ 'object_type': 'header', 'text': 'Saving changes to edited file' }) if not chosen_newline in allowed_newline.keys(): output_objects.append({ 'object_type': 'error_text', 'text': 'Unsupported newline style supplied: %s (must be one of %s)' % (chosen_newline, ', '.join(allowed_newline.keys())) }) return (output_objects, returnvalues.CLIENT_ERROR) saved_newline = allowed_newline[chosen_newline] # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing consistent # error messages abs_path = '' unfiltered_match = glob.glob(base_dir + path) for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, path)) output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % path }) return (output_objects, returnvalues.CLIENT_ERROR) if abs_path == '': # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(os.path.join(base_dir, path.lstrip(os.sep))) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, path)) output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % path }) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_path, parent_dir=True): logger.warning('%s called without write access: %s' % \ (op_name, abs_path)) output_objects.append( {'object_type': 'error_text', 'text': 'cannot edit "%s": inside a read-only location!' % \ path}) status = returnvalues.CLIENT_ERROR return (output_objects, returnvalues.CLIENT_ERROR) (owner, time_left) = acquire_edit_lock(abs_path, client_id) if owner != client_id: output_objects.append({ 'object_type': 'error_text', 'text': "You don't have the lock for %s!" % path }) return (output_objects, returnvalues.CLIENT_ERROR) try: fh = open(abs_path, 'w+') fh.write(user_arguments_dict['editarea'][0].replace( form_newline, saved_newline)) fh.close() # everything ok output_objects.append({ 'object_type': 'text', 'text': 'Saved changes to %s.' % path }) logger.info('saved changes to %s' % path) release_edit_lock(abs_path, client_id) except Exception, exc: # Don't give away information about actual fs layout output_objects.append({ 'object_type': 'error_text', 'text': '%s could not be written! (%s)' % (path, str(exc).replace(base_dir, '')) }) return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) algo_list = accepted['hash_algo'] max_chunks = int(accepted['max_chunks'][-1]) pattern_list = accepted['path'] dst = accepted['dst'][-1] current_dir = accepted['current_dir'][-1].lstrip(os.sep) # All paths are relative to current_dir pattern_list = [os.path.join(current_dir, i) for i in pattern_list] if dst: dst = os.path.join(current_dir, dst) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dir = os.path.abspath( os.path.join(base_dir, current_dir.lstrip(os.sep))) if not valid_user_path(configuration, abs_dir, base_dir, True): output_objects.append({ 'object_type': 'error_text', 'text': "You're not allowed to work in %s!" % current_dir }) logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dir, current_dir)) return (output_objects, returnvalues.CLIENT_ERROR) if verbose(flags): output_objects.append({ 'object_type': 'text', 'text': "working in %s" % current_dir }) if dst: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # NOTE: dst already incorporates current_dir prefix here # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) logger.info('chksum in %s' % abs_dest) # Don't use abs_path in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % dst }) logger.warning('%s tried to %s restricted path %s !(%s)' % (client_id, op_name, abs_dest, dst)) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_dest)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot checksum to "%s": inside a read-only location!' % relative_dest }) return (output_objects, returnvalues.CLIENT_ERROR) all_lines = [] for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') output_lines = [] for hash_algo in algo_list: try: chksum_helper = _algo_map.get(hash_algo, _algo_map["md5"]) checksum = chksum_helper(abs_path, max_chunks=max_chunks) line = "%s %s\n" % (checksum, relative_path) logger.info("%s %s of %s: %s" % (op_name, hash_algo, abs_path, checksum)) output_lines.append(line) except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = {'object_type': 'file_output', 'lines': output_lines} output_objects.append(entry) all_lines += output_lines
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) patterns = accepted['job_id'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep status = returnvalues.OK filelist = [] for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append( {'object_type': 'error_text', 'text' : '%s: You do not have any matching job IDs!' % pattern}) status = returnvalues.CLIENT_ERROR else: filelist += match # job schedule is hard on the server, limit if len(filelist) > 100: output_objects.append({'object_type': 'error_text', 'text' : 'Too many matching jobs (%s)!' % len(filelist)}) return (output_objects, returnvalues.CLIENT_ERROR) saveschedulejobs = [] for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') saveschedulejob = {'object_type': 'saveschedulejob', 'job_id': job_id} dict = unpickle(filepath, logger) if not dict: saveschedulejob['message'] = \ ('The file containing the information' \ ' for job id %s could not be opened!' \ ' You can only read schedule for ' \ 'your own jobs!') % job_id saveschedulejobs.append(saveschedulejob) status = returnvalues.CLIENT_ERROR continue saveschedulejob['oldstatus'] = dict['STATUS'] # Is the job status pending? possible_schedule_states = ['QUEUED', 'RETRY', 'FROZEN'] if not dict['STATUS'] in possible_schedule_states: saveschedulejob['message'] = \ 'You can only read schedule for jobs with status: %s.'\ % ' or '.join(possible_schedule_states) saveschedulejobs.append(saveschedulejob) continue # notify queue if not send_message_to_grid_script('JOBSCHEDULE ' + job_id + '\n', logger, configuration): output_objects.append( {'object_type': 'error_text', 'text' : 'Error sending message to grid_script, update may fail.' }) status = returnvalues.SYSTEM_ERROR continue saveschedulejobs.append(saveschedulejob) savescheduleinfo = """Please find any available job schedule status in verbose job status output.""" output_objects.append({'object_type': 'saveschedulejobs', 'saveschedulejobs': saveschedulejobs, 'savescheduleinfo': savescheduleinfo}) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) pattern_list = accepted['path'] lang = accepted['lang'][-1].lower() mode = accepted['mode'][-1] output_objects.append({'object_type': 'header', 'text' : '%s spell check' % configuration.short_title }) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK allowed_modes = ['none', 'url', 'email', 'sgml', 'tex'] # Include both base languages and variants allowed_langs = [ 'da', 'da_dk', 'de', 'en', 'en_gb', 'en_us', 'es', 'fi', 'fr', 'it', 'nl', 'no', 'se', ] # TODO: use path from settings file dict_path = '%s/%s' % (base_dir, '.personal_dictionary') if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) if not mode in allowed_modes: output_objects.append({'object_type': 'error_text', 'text' : 'Unsupported mode: %s' % mode}) status = returnvalues.CLIENT_ERROR if not lang in allowed_langs: output_objects.append({'object_type': 'error_text', 'text' : 'Unsupported lang: %s' % mode}) status = returnvalues.CLIENT_ERROR # Show all if no flags given if not flags: flags = 'blw' for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') output_lines = [] try: (out, err) = spellcheck(real_path, mode, lang, dict_path) if err: output_objects.append({'object_type': 'error_text', 'text': err}) for line in out: output_lines.append(line + '\n') except Exception, err: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, err)}) status = returnvalues.SYSTEM_ERROR continue if verbose(flags): output_objects.append({'object_type': 'file_output', 'path': relative_path, 'lines': output_lines}) else: output_objects.append({'object_type': 'file_output', 'lines': output_lines}) htmlform = \ ''' <form method="post" action="editor.py"> <input type="hidden" name="path" value="%s" /> <input type="submit" value="Edit file" /> </form> '''\ % relative_path output_objects.append({'object_type': 'html_form', 'text' : htmlform})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) dst = accepted['dst'][-1].lstrip(os.sep) pattern_list = accepted['src'] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append( {'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'Zip/tar archive extractor' output_objects.append({'object_type': 'header', 'text' : 'Zip/tar archive extractor'}) if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) if 'h' in flags: usage(output_objects) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) logger.info('unpack in %s' % abs_dest) # Don't use real dest in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): # out of bounds logger.error('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dest, dst)) output_objects.append( {'object_type': 'error_text', 'text' : "Invalid path! (%s expands to an illegal path)" % dst}) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % \ (op_name, abs_dest)) output_objects.append( {'object_type': 'error_text', 'text': 'cannot unpack to "%s": inside a read-only location!' % \ relative_dest}) return (output_objects, returnvalues.CLIENT_ERROR) status = returnvalues.OK for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file', 'name' : relative_path}) (unpack_status, msg) = unpack_archive(configuration, client_id, relative_path, relative_dest) if not unpack_status: output_objects.append({'object_type': 'error_text', 'text': 'Error: %s' % msg}) status = returnvalues.CLIENT_ERROR continue output_objects.append( {'object_type': 'text', 'text' : 'The zip/tar archive %s was unpacked in %s' % (relative_path, relative_dest)}) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) path = accepted['path'][-1] chosen_newline = accepted['newline'][-1] submitjob = accepted['submitjob'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep # HTML spec dictates newlines in forms to be MS style (\r\n) # rather than un*x style (\n): change if requested. form_newline = '\r\n' allowed_newline = {'unix': '\n', 'mac': '\r', 'windows': '\r\n'} output_objects.append({'object_type': 'header', 'text' : 'Saving changes to edited file'}) if not chosen_newline in allowed_newline.keys(): output_objects.append( {'object_type': 'error_text', 'text' : 'Unsupported newline style supplied: %s (must be one of %s)' % (chosen_newline, ', '.join(allowed_newline.keys()))}) return (output_objects, returnvalues.CLIENT_ERROR) saved_newline = allowed_newline[chosen_newline] # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing consistent # error messages real_path = '' unfiltered_match = glob.glob(base_dir + path) for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, path)) output_objects.append( {'object_type': 'error_text', 'text' : "Invalid path! (%s expands to an illegal path)" % path}) return (output_objects, returnvalues.CLIENT_ERROR) if real_path == '': real_path = base_dir + path if not valid_user_path(real_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, path)) output_objects.append( {'object_type': 'error_text', 'text' : "Invalid path! (%s expands to an illegal path)" % path}) return (output_objects, returnvalues.CLIENT_ERROR) (owner, time_left) = acquire_edit_lock(real_path, client_id) if owner != client_id: output_objects.append({'object_type': 'error_text', 'text' : "You don't have the lock for %s!" % path}) return (output_objects, returnvalues.CLIENT_ERROR) try: fh = open(real_path, 'w+') fh.write(user_arguments_dict['editarea' ][0].replace(form_newline, saved_newline)) fh.close() # everything ok output_objects.append({'object_type': 'text', 'text' : 'Saved changes to %s.' % path}) release_edit_lock(real_path, client_id) except Exception, exc: # Don't give away information about actual fs layout output_objects.append({'object_type': 'error_text', 'text' : '%s could not be written! (%s)' % (path, str(exc).replace(base_dir, '' ))}) return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) defaults = signature()[1] output_objects.append({ 'object_type': 'header', 'text': 'Reject Resource Request' }) (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) unique_resource_name = accepted['unique_resource_name'][-1].strip() request_name = unhexlify(accepted['request_name'][-1]) if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not is_owner(client_id, unique_resource_name, configuration.resource_home, logger): output_objects.append({ 'object_type': 'error_text', 'text': 'You must be an owner of %s to reject requests!' % unique_resource_name }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # resource dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.resource_home, unique_resource_name)) + os.sep # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(os.path.join(base_dir, request_name)) if not valid_user_path( configuration, abs_path, base_dir, allow_equal=False): logger.warning('%s tried to access restricted path %s ! (%s)' % \ (client_id, abs_path, request_name)) output_objects.append({ 'object_type': 'error_text', 'text': '''Illegal request name "%s": you can only reject requests to your own resources.''' % request_name }) return (output_objects, returnvalues.CLIENT_ERROR) if request_name: request_dir = os.path.join(configuration.resource_home, unique_resource_name) req = load_access_request(configuration, request_dir, request_name) if not req or not delete_access_request(configuration, request_dir, request_name): logger.error("failed to delete owner request for %s in %s" % \ (unique_resource_name, request_name)) output_objects.append({ 'object_type': 'error_text', 'text': 'Failed to remove saved resource request for %s in %s!'\ % (unique_resource_name, request_name)}) return (output_objects, returnvalues.CLIENT_ERROR) output_objects.append({ 'object_type': 'text', 'text': ''' Deleted %(request_type)s access request to %(target)s for %(entity)s . ''' % req }) form_method = 'post' csrf_limit = get_csrf_limit(configuration) fill_helpers = { 'protocol': any_protocol, 'unique_resource_name': unique_resource_name, 'form_method': form_method, 'csrf_field': csrf_field, 'csrf_limit': csrf_limit } fill_helpers.update(req) target_op = 'sendrequestaction' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token}) output_objects.append({ 'object_type': 'html_form', 'text': """ <p> You can use the reply form below if you want to additionally send an explanation for rejecting the request. </p> <form method='%(form_method)s' action='%(target_op)s.py'> <input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' /> <input type=hidden name=request_type value='resourcereject' /> <input type=hidden name=unique_resource_name value='%(target)s' /> <input type=hidden name=cert_id value='%(entity)s' /> <input type=hidden name=protocol value='%(protocol)s' /> <table> <tr> <td class='title'>Optional reject message to requestor(s)</td> </tr><tr> <td><textarea name=request_text cols=72 rows=10> We have decided to reject your %(request_type)s request to our %(target)s resource. Regards, the %(target)s resource owners </textarea></td> </tr> <tr> <td><input type='submit' value='Inform requestor(s)' /></td> </tr> </table> </form> <br /> """ % fill_helpers }) output_objects.append({ 'object_type': 'link', 'destination': 'resadmin.py?unique_resource_name=%s' % unique_resource_name, 'text': 'Back to administration for %s' % unique_resource_name }) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input(user_arguments_dict, defaults, output_objects, allow_rejects=False) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) pattern_list = accepted['path'] iosessionid = accepted['iosessionid'][-1] if not client_id: if not iosessionid.strip() or not iosessionid.isalnum(): # deny output_objects.append( {'object_type': 'error_text', 'text' : 'No sessionid or invalid sessionid supplied!'}) return (output_objects, returnvalues.CLIENT_ERROR) base_dir_no_sessionid = \ os.path.realpath(configuration.webserver_home) + os.sep base_dir = \ os.path.realpath(os.path.join(configuration.webserver_home, iosessionid)) + os.sep if not os.path.isdir(base_dir): # deny output_objects.append({'object_type': 'error_text', 'text' : 'Invalid sessionid!'}) return (output_objects, returnvalues.CLIENT_ERROR) if not valid_user_path(base_dir, base_dir_no_sessionid, True): # deny output_objects.append({'object_type': 'error_text', 'text' : 'Invalid sessionid!'}) return (output_objects, returnvalues.CLIENT_ERROR) else: # TODO: this is a hack to allow truncate - fix 'put' empty files # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! ( %s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file', 'name' : relative_path}) # Make it harder to accidentially delete too much - e.g. do not delete # VGrid files without explicit selection of subdir contents if real_path == os.path.abspath(base_dir): output_objects.append({'object_type': 'warning', 'text' : "You're not allowed to delete your entire home directory!" }) status = returnvalues.CLIENT_ERROR continue if os.path.islink(real_path): output_objects.append({'object_type': 'warning', 'text' : "You're not allowed to delete entire %s shared dirs!" % configuration.site_vgrid_label }) status = returnvalues.CLIENT_ERROR continue if os.path.isdir(real_path) and recursive(flags): # bottom up traversal of the file tree since rmdir is limited to # empty dirs for (root, dirs, files) in os.walk(real_path, topdown=False): for name in files: path = os.path.join(root, name) relative_path = path.replace(base_dir, '') # Traversal may find additional invisible files to skip if invisible_file(name): continue if verbose(flags): output_objects.append({'object_type': 'file' , 'name': relative_path}) try: os.remove(path) except Exception, exc: output_objects.append({'object_type' : 'error_text', 'text' : "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR for name in dirs: path = os.path.join(root, name) relative_path = path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file' , 'name': relative_path}) try: os.rmdir(path) except Exception, exc: output_objects.append({'object_type' : 'error_text', 'text' : "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR # Finally remove base directory relative_path = real_path.replace(base_dir, '') try: os.rmdir(real_path) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') output_lines = [] try: filedes = open(abs_path, 'r') i = 0 for line in filedes: if i >= lines: break output_lines.append(line) i += 1 filedes.close() except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = { 'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines'] } if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] name_pattern = accepted['name'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) dir_listings = [] output_objects.append({ 'object_type': 'dir_listings', 'dir_listings': dir_listings, 'flags': flags, }) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') entries = [] dir_listing = { 'object_type': 'dir_listing', 'relative_path': relative_path, 'entries': entries, 'flags': flags, } dir_listings.append(dir_listing) try: for (root, dirs, files) in os.walk(abs_path): for filename in fnmatch.filter(files, name_pattern): # IMPORTANT: this join always yields abs expanded path abs_path = os.path.join(root, filename) relative_path = abs_path.replace(base_dir, '') if not valid_user_path(configuration, abs_path, base_dir, True): continue file_obj = { 'object_type': 'direntry', 'type': 'file', 'name': filename, 'file_with_dir': relative_path, 'flags': flags, 'special': '', } entries.append(file_obj) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if verbose(flags): output_objects.append({'object_type': 'file_output', 'path': relative_path, 'lines': output_lines}) else: output_objects.append({'object_type': 'file_output', 'lines': output_lines})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) # Show all if no type flags given if not byte_count(flags) and not line_count(flags)\ and not word_count(flags): flags += 'blw' for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND filewcs = [] for abs_path in match: relative_path = abs_path.replace(base_dir, '') (bytes, words, lines) = (0, 0, 0) try: obj = {'object_type': 'filewc', 'name': relative_path} if os.path.isdir(abs_path): obj = { 'object_type': 'filewc', 'name': '%s: %s: Is a directory' % (op_name, relative_path), 'lines': 0, 'words': 0, 'bytes': 0, } filewcs.append(obj) continue fd = open(abs_path, 'r') lines = 0 # use file directly as iterator for efficiency for line in fd: lines += 1 bytes += len(line) words += len(line.split()) if line_count(flags): obj['lines'] = lines if word_count(flags): obj['words'] = words if byte_count(flags): obj['bytes'] = bytes filewcs.append(obj) except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'filewcs', 'filewcs': filewcs})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') output_lines = [] try: filedes = open(real_path, 'r') i = 0 for line in filedes: if i >= lines: break output_lines.append(line) i += 1 filedes.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = {'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines']} if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = accepted['flags'] patterns = accepted['job_id'] if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep mrsl_keywords_dict = get_keywords_dict(configuration) if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in patterns: # Add file extension pattern += '.mRSL' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') try: mrsl_dict = unpickle(abs_path, logger) if not mrsl_dict: raise Exception('could not load job mRSL') for (key, val) in mrsl_dict.items(): if not key in mrsl_keywords_dict.keys(): continue if not val: continue output_lines.append('::%s::\n' % key) if 'multiplestrings' == mrsl_keywords_dict[key]['Type']: for line in val: output_lines.append('%s\n' % line) elif 'multiplekeyvalues' == mrsl_keywords_dict[key][ 'Type']: for (left, right) in val: output_lines.append('%s=%s\n' % (left, right)) else: output_lines.append('%s\n' % val) output_lines.append('\n') except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if verbose(flags): output_objects.append({ 'object_type': 'file_output', 'path': relative_path, 'lines': output_lines }) else: output_objects.append({ 'object_type': 'file_output', 'lines': output_lines })
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: logger.error("jobstatus input validation failed: %s" % accepted) return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) max_jobs = int(accepted['max_jobs'][-1]) order = 'unsorted ' if sorted(flags): order = 'sorted ' patterns = accepted['job_id'] project_names = accepted['project_name'] if len(project_names) > 0: for project_name in project_names: project_name_job_ids = \ get_job_ids_with_specified_project_name(client_id, project_name, configuration.mrsl_files_dir, logger) patterns.extend(project_name_job_ids) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep output_objects.append({'object_type': 'header', 'text' : '%s %s job status' % \ (configuration.short_title, order)}) if not patterns: output_objects.append({'object_type': 'error_text', 'text' : 'No job_id specified!'}) return (output_objects, returnvalues.NO_SUCH_JOB_ID) if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) if not os.path.isdir(base_dir): output_objects.append( {'object_type': 'error_text', 'text' : ('You have not been created as a user on the %s server! ' \ 'Please contact the %s team.') % \ (configuration.short_title, configuration.short_title)}) return (output_objects, returnvalues.CLIENT_ERROR) filelist = [] for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to # avoid leaking information about file system layout while # allowing consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match.... if not match: output_objects.append( {'object_type': 'error_text', 'text' : '%s: You do not have any matching job IDs!' % pattern}) status = returnvalues.CLIENT_ERROR else: filelist += match if sorted(flags): sort(filelist) if max_jobs < len(filelist): output_objects.append( {'object_type': 'text', 'text' : 'Only showing first %d of the %d matching jobs as requested' % (max_jobs, len(filelist))}) filelist = filelist[:max_jobs] # Iterate through jobs and print details for each job_list = {'object_type': 'job_list', 'jobs': []} for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') job_dict = unpickle(filepath, logger) if not job_dict: status = returnvalues.CLIENT_ERROR output_objects.append( {'object_type': 'error_text', 'text' : 'No such job: %s (could not load mRSL file %s)' % \ (job_id, filepath)}) continue # Expand any job variables before use job_dict = expand_variables(job_dict) job_obj = {'object_type': 'job', 'job_id': job_id} job_obj['status'] = job_dict['STATUS'] time_fields = [ 'VERIFIED', 'VERIFIED_TIMESTAMP', 'RECEIVED_TIMESTAMP', 'QUEUED_TIMESTAMP', 'SCHEDULE_TIMESTAMP', 'EXECUTING_TIMESTAMP', 'FINISHED_TIMESTAMP', 'FAILED_TIMESTAMP', 'CANCELED_TIMESTAMP', ] for name in time_fields: if job_dict.has_key(name): # time objects cannot be marshalled, asctime if timestamp try: job_obj[name.lower()] = time.asctime(job_dict[name]) except Exception, exc: # not a time object, just add job_obj[name.lower()] = job_dict[name] ########################################### # ARC job status retrieval on demand: # But we should _not_ update the status in the mRSL files, since # other MiG code might rely on finding only valid "MiG" states. if configuration.arc_clusters and \ job_dict.get('UNIQUE_RESOURCE_NAME', 'unset') == 'ARC' \ and job_dict['STATUS'] == 'EXECUTING': try: home = os.path.join(configuration.user_home, client_dir) arcsession = arc.Ui(home) arcstatus = arcsession.jobStatus(job_dict['EXE']) job_obj['status'] = arcstatus['status'] except arc.ARCWrapperError, err: logger.error('Error retrieving ARC job status: %s' % \ err.what()) job_obj['status'] += '(Error: ' + err.what() + ')' except arc.NoProxyError, err: logger.error('While retrieving ARC job status: %s' % \ err.what()) job_obj['status'] += '(Error: ' + err.what() + ')'
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) path_list = accepted['path'] size_list = [int(size) for size in accepted['size']] title_entry = find_entry(output_objects, 'title') title_entry['text'] = '%s Upload Progress Monitor' % configuration.short_title if not configuration.site_enable_griddk: output_objects.append({'object_type': 'text', 'text': '''Grid.dk features are disabled on this site. Please contact the Grid admins %s if you think they should be enabled. ''' % configuration.admin_email}) return (output_objects, returnvalues.OK) refresh_secs = 5 meta = '<meta http-equiv="refresh" content="%s" />' % refresh_secs title_entry['meta'] = meta # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep output_objects.append({'object_type': 'header', 'text' : 'Upload progress'}) done_list = [False for _ in path_list] progress_items = [] index = -1 for path in path_list: index += 1 # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages real_path = os.path.abspath(os.path.join(base_dir, path)) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, path)) output_objects.append({'object_type': 'file_not_found', 'name': path}) status = returnvalues.FILE_NOT_FOUND continue if not os.path.isfile(real_path): output_objects.append({'object_type': 'error_text', 'text' : "no such upload: %s" % path}) status = returnvalues.CLIENT_ERROR continue relative_path = real_path.replace(base_dir, '') try: logger.info('Checking size of upload %s' % real_path) cur_size = os.path.getsize(real_path) total_size = size_list[index] percent = round(100.0 * cur_size / total_size, 1) done_list[index] = (cur_size == total_size) progress_items.append({'object_type': 'progress', 'path': path, 'cur_size': cur_size, 'total_size': total_size, 'percent': percent, 'done': done_list[index], 'progress_type': 'upload'}) except Exception, exc: # Don't give away information about actual fs layout output_objects.append({'object_type': 'error_text', 'text' : '%s upload could not be checked! (%s)' % (path, str(exc).replace(base_dir, '' ))}) status = returnvalues.SYSTEM_ERROR continue
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) dst = accepted['dst'][-1].lstrip(os.sep) pattern_list = accepted['src'] current_dir = accepted['current_dir'][-1].lstrip(os.sep) # All paths are relative to current_dir pattern_list = [os.path.join(current_dir, i) for i in pattern_list] dst = os.path.join(current_dir, dst) if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append( {'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'ZIP/TAR archiver' output_objects.append( {'object_type': 'header', 'text': 'ZIP/TAR archiver'}) if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag)}) if 'h' in flags: usage(output_objects) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dir = os.path.abspath(os.path.join(base_dir, current_dir.lstrip(os.sep))) if not valid_user_path(configuration, abs_dir, base_dir, True): # out of bounds output_objects.append({'object_type': 'error_text', 'text': "You're not allowed to work in %s!" % current_dir}) logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dir, current_dir)) return (output_objects, returnvalues.CLIENT_ERROR) output_objects.append( {'object_type': 'text', 'text': "working in %s" % current_dir}) # NOTE: dst already incorporates current_dir prefix here # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) logger.info('pack in %s' % abs_dest) # Don't use abs_path in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): # out of bounds output_objects.append( {'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % dst}) logger.warning('%s tried to %s restricted path %s !(%s)' % (client_id, op_name, abs_dest, dst)) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(os.path.dirname(abs_dest)): output_objects.append({'object_type': 'error_text', 'text': "No such destination directory: %s" % os.path.dirname(relative_dest)}) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_dest)) output_objects.append( {'object_type': 'error_text', 'text': 'cannot pack to "%s": inside a read-only location!' % relative_dest}) return (output_objects, returnvalues.CLIENT_ERROR) status = returnvalues.OK for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: logger.debug("%s: inspecting: %s" % (op_name, server_path)) # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue else: match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'error_text', 'text': "%s: cannot pack '%s': no valid src paths" % (op_name, pattern)}) status = returnvalues.CLIENT_ERROR continue for abs_path in match: relative_path = abs_path.replace(base_dir, '') # Generally refuse handling symlinks including root vgrid shares if os.path.islink(abs_path): output_objects.append( {'object_type': 'warning', 'text': """You're not allowed to pack entire special folders like %s shared folders!""" % configuration.site_vgrid_label}) status = returnvalues.CLIENT_ERROR continue # Additionally refuse operations on inherited subvgrid share roots elif in_vgrid_share(configuration, abs_path) == relative_path: output_objects.append( {'object_type': 'warning', 'text': """You're not allowed to pack entire %s shared folders!""" % configuration.site_vgrid_label}) status = returnvalues.CLIENT_ERROR continue elif os.path.realpath(abs_path) == os.path.realpath(base_dir): logger.error("%s: refusing pack home dir: %s" % (op_name, abs_path)) output_objects.append( {'object_type': 'warning', 'text': "You're not allowed to pack your entire home directory!" }) status = returnvalues.CLIENT_ERROR continue elif os.path.realpath(abs_dest) == os.path.realpath(abs_path): output_objects.append({'object_type': 'warning', 'text': 'overlap in source and destination %s' % relative_dest}) status = returnvalues.CLIENT_ERROR continue if verbose(flags): output_objects.append( {'object_type': 'file', 'name': relative_path}) (pack_status, msg) = pack_archive(configuration, client_id, relative_path, relative_dest) if not pack_status: output_objects.append({'object_type': 'error_text', 'text': 'Error: %s' % msg}) status = returnvalues.CLIENT_ERROR continue output_objects.append({'object_type': 'text', 'text': 'Added %s to %s' % (relative_path, relative_dest)}) output_objects.append({'object_type': 'text', 'text': 'Packed archive of %s is now available in %s' % (', '.join(pattern_list), relative_dest)}) output_objects.append({'object_type': 'link', 'text': 'Download archive', 'destination': os.path.join('..', client_dir, relative_dest)}) return (output_objects, status)
dest = dst_list[-1] if len(dst_list) > 1: output_objects.append( {'object_type': 'warning', 'text': 'dst (%s) matches multiple targets - using last: %s' % (dst, dest)}) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(dest) # Don't use abs_path in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(dst_base, '') if not valid_user_path(configuration, abs_dest, dst_base, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dest, dst)) output_objects.append( {'object_type': 'error_text', 'text': "Invalid destination (%s expands to an illegal path)" % dst}) return (output_objects, returnvalues.CLIENT_ERROR) # We must make sure target dir exists if called in import X mode if (share_id or freeze_id) and not makedirs_rec(abs_dest, configuration): logger.error('could not create import destination dir: %s' % abs_dest) output_objects.append( {'object_type': 'error_text', 'text': 'cannot import to "%s" : file in the way?' % relative_dest}) return (output_objects, returnvalues.SYSTEM_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s'
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] status = returnvalues.OK (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] dst = accepted['dst'][-1].lstrip(os.sep) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) if dst: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) dst_mode = "wb" # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) relative_dst = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): logger.warning('%s tried to %s into restricted path %s ! (%s)' % (client_id, op_name, abs_dest, dst)) output_objects.append({ 'object_type': 'error_text', 'text': "invalid destination: '%s'" % dst }) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'accessed', [relative_path]) fd = open(abs_path, 'r') # use file directly as iterator for efficiency for line in fd: output_lines.append(line) fd.close() except Exception, exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'accessed', [relative_path], failed=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if dst: try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'modified', [dst]) out_fd = open(abs_dest, dst_mode) out_fd.writelines(output_lines) out_fd.close() logger.info('%s %s %s done' % (op_name, abs_path, abs_dest)) except Exception, exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'modified', [dst], error=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "write failed: '%s'" % exc }) logger.error("%s: write failed on '%s': %s" % (op_name, abs_dest, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({ 'object_type': 'text', 'text': "wrote %s to %s" % (relative_path, relative_dst) }) # Prevent truncate after first write dst_mode = "ab+" else: entry = { 'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines'] } if verbose(flags): entry['path'] = relative_path output_objects.append(entry) # TODO: rip this hack out into real download handler? # Force download of files when output_format == 'file_format' # This will only work for the first file matching a glob when # using file_format. # And it is supposed to only work for one file. if user_arguments_dict.has_key('output_format'): output_format = user_arguments_dict['output_format'][0] if output_format == 'file': output_objects.append({ 'object_type': 'start', 'headers': [('Content-Disposition', 'attachment; filename="%s";' % os.path.basename(abs_path))] })
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) patterns = accepted['job_id'] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep status = returnvalues.OK filelist = [] for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % pattern }) status = returnvalues.CLIENT_ERROR else: filelist += match # job schedule is hard on the server, limit if len(filelist) > 100: output_objects.append({ 'object_type': 'error_text', 'text': 'Too many matching jobs (%s)!' % len(filelist) }) return (output_objects, returnvalues.CLIENT_ERROR) saveschedulejobs = [] for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') saveschedulejob = {'object_type': 'saveschedulejob', 'job_id': job_id} dict = unpickle(filepath, logger) if not dict: saveschedulejob['message'] = \ ('The file containing the information' \ ' for job id %s could not be opened!' \ ' You can only read schedule for ' \ 'your own jobs!') % job_id saveschedulejobs.append(saveschedulejob) status = returnvalues.CLIENT_ERROR continue saveschedulejob['oldstatus'] = dict['STATUS'] # Is the job status pending? possible_schedule_states = ['QUEUED', 'RETRY', 'FROZEN'] if not dict['STATUS'] in possible_schedule_states: saveschedulejob['message'] = \ 'You can only read schedule for jobs with status: %s.'\ % ' or '.join(possible_schedule_states) saveschedulejobs.append(saveschedulejob) continue # notify queue if not send_message_to_grid_script('JOBSCHEDULE ' + job_id + '\n', logger, configuration): output_objects.append({ 'object_type': 'error_text', 'text': 'Error sending message to grid_script, update may fail.' }) status = returnvalues.SYSTEM_ERROR continue saveschedulejobs.append(saveschedulejob) savescheduleinfo = """Please find any available job schedule status in verbose job status output.""" output_objects.append({ 'object_type': 'saveschedulejobs', 'saveschedulejobs': saveschedulejobs, 'savescheduleinfo': savescheduleinfo }) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') output_lines = [] # We search for the last 'lines' lines by beginning from the end of # the file and exponetially backtracking until the backtrack # contains at least 'lines' lines or we're back to the beginning of # the file. # At that point we skip any extra lines before printing. try: filedes = open(real_path, 'r') # Go to end of file and backtrack backstep = 1 newlines = 0 filedes.seek(0, 2) length = filedes.tell() while backstep < length and newlines <= lines: backstep *= 2 if backstep > length: backstep = length filedes.seek(-backstep, 2) newlines = len(filedes.readlines()) if length: # Go back after reading to end of file filedes.seek(-backstep, 2) # Skip any extra lines caused by the exponential backtracking. # We could probably speed up convergence with binary search... while newlines > lines: dummy = filedes.readline() newlines -= 1 backstep = length - filedes.tell() # Now we're at the wanted spot - print rest of file for _ in range(newlines): output_lines.append(filedes.readline()) filedes.close() except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = {'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines']} if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) # Show all if no type flags given if not byte_count(flags) and not line_count(flags)\ and not word_count(flags): flags += 'blw' for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND filewcs = [] for real_path in match: relative_path = real_path.replace(base_dir, '') (bytes, words, lines) = (0, 0, 0) try: obj = {'object_type': 'filewc', 'name': relative_path} if os.path.isdir(real_path): obj = { 'object_type': 'filewc', 'name': '%s: %s: Is a directory' % (op_name, relative_path), 'lines': 0, 'words': 0, 'bytes': 0, } filewcs.append(obj) continue fd = open(real_path, 'r') lines = 0 # use file directly as iterator for efficiency for line in fd: lines += 1 bytes += len(line) words += len(line.split()) if line_count(flags): obj['lines'] = lines if word_count(flags): obj['words'] = words if byte_count(flags): obj['bytes'] = bytes filewcs.append(obj) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'filewcs', 'filewcs' : filewcs})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) # TODO: if validator is too tight we should accept rejects here # and then make sure that such rejected fields are never printed if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) path = accepted['path'][-1] current_dir = accepted['current_dir'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep # the client can choose to specify the path of the target directory with # current_dir + "/" + path, instead of specifying the complete path in # subdirs. This is usefull from ls.py where a hidden html control makes it # possible to target the directory from the current dir. title_entry = find_entry(output_objects, 'title') title_entry['text'] = '%s file web editor' % configuration.short_title title_entry['style'] = advanced_editor_css_deps() title_entry['javascript'] = advanced_editor_js_deps() title_entry['javascript'] += lock_info('this file', -1) output_objects.append({'object_type': 'header', 'text' : 'Editing file in %s home directory' % \ configuration.short_title }) if not path: now = time.gmtime() path = 'noname-%s.txt' % time.strftime('%d%m%y-%H%M%S', now) output_objects.append({'object_type': 'text', 'text' : 'No path supplied - creating new file in %s' % path}) path = os.path.normpath(current_dir + path) real_path = os.path.abspath(base_dir + current_dir + path) if not valid_user_path(real_path, base_dir): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, path)) output_objects.append( {'object_type': 'error_text', 'text' : "Invalid path! (%s expands to an illegal path)" % path}) return (output_objects, returnvalues.CLIENT_ERROR) (owner, time_left) = acquire_edit_lock(real_path, client_id) if owner == client_id: javascript = \ '''<script type="text/javascript"> setTimeout("newcountdown('%s', %d)", 1) </script> '''\ % (path, time_left / 60) output_objects.append({'object_type': 'html_form', 'text' : javascript}) html = edit_file(path, real_path) output_objects.append({'object_type': 'html_form', 'text' : html}) else: output_objects.append( {'object_type': 'error_text', 'text' : '%s acquired the editing lock for %s! (timeout in %d seconds)' % (owner, path, time_left)}) return (output_objects, returnvalues.CLIENT_ERROR) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] # IMPORTANT: the CGI front end forces the input extraction to be delayed # We must manually extract and parse input here to avoid memory explosion # for huge files! # TODO: explosions still happen sometimes! # Most likely because of Apache SSL renegotiations which have # no other way of storing input extract_input = user_arguments_dict.get('__DELAYED_INPUT__', dict) logger.info('Extracting input in %s' % op_name) form = extract_input() logger.info('After extracting input in %s' % op_name) file_item = None file_name = '' user_arguments_dict = {} if form.has_key('fileupload'): file_item = form['fileupload'] file_name = file_item.filename user_arguments_dict['fileupload'] = ['true'] user_arguments_dict['path'] = [file_name] if form.has_key('path'): user_arguments_dict['path'] = [form['path'].value] if form.has_key('restrict'): user_arguments_dict['restrict'] = [form['restrict'].value] else: user_arguments_dict['restrict'] = defaults['restrict'] logger.info('Filtered input is: %s' % user_arguments_dict) # Now validate parts as usual (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) path = accepted['path'][-1] restrict = accepted['restrict'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_griddk: output_objects.append({ 'object_type': 'text', 'text': '''Grid.dk features are disabled on this site. Please contact the site admins %s if you think they should be enabled. ''' % configuration.admin_email }) return (output_objects, returnvalues.OK) logger.info('Filtered input validated with result: %s' % accepted) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) output_objects.append({'object_type': 'header', 'text': 'Uploading file'}) # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing consistent # error messages real_path = os.path.realpath(os.path.join(base_dir, path)) # Implicit destination if os.path.isdir(real_path): real_path = os.path.join(real_path, os.path.basename(file_name)) if not valid_user_path(configuration, real_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, path)) output_objects.append({ 'object_type': 'error_text', 'text': "Invalid destination (%s expands to an illegal path)" % path }) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(os.path.dirname(real_path)): output_objects.append({ 'object_type': 'error_text', 'text': "cannot write: no such file or directory: %s)" % path }) return (output_objects, returnvalues.CLIENT_ERROR) # We fork off here and redirect the user to a progress page for user # friendly output and to avoid cgi timeouts from killing the upload. # We use something like the Active State python recipe for daemonizing # to properly detach from the CGI process and continue in the background. # Please note that we only close stdio file descriptors to avoid closing # the fileupload. file_item.file.seek(0, 2) total_size = file_item.file.tell() file_item.file.seek(0, 0) try: pid = os.fork() if pid == 0: os.setsid() pid = os.fork() if pid == 0: os.chdir('/') os.umask(0) for fno in range(3): try: os.close(fno) except OSError: pass else: os._exit(0) except OSError, ose: output_objects.append({ 'object_type': 'error_text', 'text': '%s upload could not background! (%s)' % (path, str(ose).replace(base_dir, '')) }) return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] # IMPORTANT: the CGI front end forces the input extraction to be delayed # We must manually extract and parse input here to avoid memory explosion # for huge files! # TODO: explosions still happen sometimes! # Most likely because of Apache SSL renegotiations which have # no other way of storing input extract_input = user_arguments_dict["__DELAYED_INPUT__"] logger.info("Extracting input in %s" % op_name) form = extract_input() logger.info("After extracting input in %s" % op_name) file_item = None file_name = "" user_arguments_dict = {} if form.has_key("fileupload"): file_item = form["fileupload"] file_name = file_item.filename user_arguments_dict["fileupload"] = ["true"] user_arguments_dict["path"] = [file_name] if form.has_key("path"): user_arguments_dict["path"] = [form["path"].value] if form.has_key("restrict"): user_arguments_dict["restrict"] = [form["restrict"].value] else: user_arguments_dict["restrict"] = defaults["restrict"] logger.info("Filtered input is: %s" % user_arguments_dict) # Now validate parts as usual (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler("POST"): output_objects.append( {"object_type": "error_text", "text": "Only accepting POST requests to prevent unintended updates"} ) return (output_objects, returnvalues.CLIENT_ERROR) flags = "".join(accepted["flags"]) path = accepted["path"][-1] restrict = accepted["restrict"][-1] if not configuration.site_enable_griddk: output_objects.append( { "object_type": "text", "text": """Grid.dk features are disabled on this site. Please contact the Grid admins %s if you think they should be enabled. """ % configuration.admin_email, } ) return (output_objects, returnvalues.OK) logger.info("Filtered input validated with result: %s" % accepted) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({"object_type": "text", "text": "%s using flag: %s" % (op_name, flag)}) output_objects.append({"object_type": "header", "text": "Uploading file"}) # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing consistent # error messages real_path = os.path.realpath(os.path.join(base_dir, path)) # Implicit destination if os.path.isdir(real_path): real_path = os.path.join(real_path, os.path.basename(file_name)) if not valid_user_path(real_path, base_dir, True): logger.warning("%s tried to %s restricted path %s ! (%s)" % (client_id, op_name, real_path, path)) output_objects.append( {"object_type": "error_text", "text": "Invalid destination (%s expands to an illegal path)" % path} ) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(os.path.dirname(real_path)): output_objects.append( {"object_type": "error_text", "text": "cannot write: no such file or directory: %s)" % path} ) return (output_objects, returnvalues.CLIENT_ERROR) # We fork off here and redirect the user to a progress page for user # friendly output and to avoid cgi timeouts from killing the upload. # We use something like the Active State python recipe for daemonizing # to properly detach from the CGI process and continue in the background. # Please note that we only close stdio file descriptors to avoid closing # the fileupload. file_item.file.seek(0, 2) total_size = file_item.file.tell() file_item.file.seek(0, 0) try: pid = os.fork() if pid == 0: os.setsid() pid = os.fork() if pid == 0: os.chdir("/") os.umask(0) for fno in range(3): try: os.close(fno) except OSError: pass else: os._exit(0) except OSError, ose: output_objects.append( { "object_type": "error_text", "text": "%s upload could not background! (%s)" % (path, str(ose).replace(base_dir, "")), } ) return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) src_list = accepted['src'] dst = accepted['dst'][-1] iosessionid = accepted['iosessionid'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if not client_id: base_dir = os.path.realpath(configuration.webserver_home + os.sep + iosessionid) + os.sep status = returnvalues.OK real_dest = base_dir + dst dst_list = glob.glob(real_dest) if not dst_list: # New destination? if not glob.glob(os.path.dirname(real_dest)): output_objects.append({'object_type': 'error_text', 'text' : 'Illegal dst path provided!'}) return (output_objects, returnvalues.CLIENT_ERROR) else: dst_list = [real_dest] # Use last match in case of multiple matches dest = dst_list[-1] if len(dst_list) > 1: output_objects.append( {'object_type': 'warning', 'text' : 'dst (%s) matches multiple targets - using last: %s' % (dst, dest)}) real_dest = os.path.abspath(dest) # Don't use real_path in output as it may expose underlying # fs layout. relative_dest = real_dest.replace(base_dir, '') if not valid_user_path(real_dest, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_dest, dst)) output_objects.append( {'object_type': 'error_text', 'text' : "Invalid destination (%s expands to an illegal path)" % dst}) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in src_list: unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: relative_path = real_path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file', 'name' : relative_path}) # src must be a file unless recursive is specified if not recursive(flags) and os.path.isdir(real_path): logger.warning('skipping directory source %s' % real_path) output_objects.append({'object_type': 'warning', 'text' : 'skipping directory src %s!' % relative_path}) continue # If destination is a directory the src should be copied there real_target = real_dest if os.path.isdir(real_target): real_target = os.path.join(real_target, os.path.basename(real_path)) if os.path.abspath(real_path) == os.path.abspath(real_target): logger.warning('%s tried to %s %s to itself! (%s)' % \ (client_id, op_name, real_path, pattern)) output_objects.append( {'object_type': 'warning', 'text' : "Cannot copy '%s' to self!" % relative_path}) status = returnvalues.CLIENT_ERROR continue if os.path.isdir(real_path) and \ real_target.startswith(real_path + os.sep): logger.warning('%s tried to %s %s to itself! (%s)' % (client_id, op_name, real_path, pattern)) output_objects.append( {'object_type': 'warning', 'text' : "Cannot copy '%s' to (sub) self!" % relative_path}) status = returnvalues.CLIENT_ERROR continue try: if os.path.isdir(real_path): shutil.copytree(real_path, real_target) else: shutil.copy(real_path, real_target) logger.info('%s %s %s done' % (op_name, real_path, real_target)) except Exception, exc: output_objects.append( {'object_type': 'error_text', 'text': "%s: failed on '%s' to '%s'" \ % (op_name, relative_path, relative_dest)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) patterns = accepted['job_id'] action = accepted['action'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) if not action in valid_actions.keys(): output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid job action "%s" (only %s supported)' % (action, ', '.join(valid_actions.keys())) }) return (output_objects, returnvalues.CLIENT_ERROR) new_state = valid_actions[action] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep status = returnvalues.OK filelist = [] for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.error( '%s tried to use %s %s outside own home! (pattern %s)' % (client_id, op_name, abs_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % pattern }) status = returnvalues.CLIENT_ERROR else: filelist += match # job state change is hard on the server, limit if len(filelist) > 500: output_objects.append({ 'object_type': 'error_text', 'text': 'Too many matching jobs (%s)!' % len(filelist) }) return (output_objects, returnvalues.CLIENT_ERROR) changedstatusjobs = [] for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') changedstatusjob = { 'object_type': 'changedstatusjob', 'job_id': job_id } job_dict = unpickle(filepath, logger) if not job_dict: changedstatusjob['message'] = '''The file containing the information for job id %s could not be opened! You can only %s your own jobs!''' % (job_id, action) changedstatusjobs.append(changedstatusjob) status = returnvalues.CLIENT_ERROR continue changedstatusjob['oldstatus'] = job_dict['STATUS'] # Is the job status compatible with action? possible_cancel_states = [ 'PARSE', 'QUEUED', 'RETRY', 'EXECUTING', 'FROZEN' ] if action == 'cancel' and \ not job_dict['STATUS'] in possible_cancel_states: changedstatusjob['message'] = \ 'You can only cancel jobs with status: %s.'\ % ' or '.join(possible_cancel_states) status = returnvalues.CLIENT_ERROR changedstatusjobs.append(changedstatusjob) continue possible_freeze_states = ['QUEUED', 'RETRY'] if action == 'freeze' and \ not job_dict['STATUS'] in possible_freeze_states: changedstatusjob['message'] = \ 'You can only freeze jobs with status: %s.'\ % ' or '.join(possible_freeze_states) status = returnvalues.CLIENT_ERROR changedstatusjobs.append(changedstatusjob) continue possible_thaw_states = ['FROZEN'] if action == 'thaw' and \ not job_dict['STATUS'] in possible_thaw_states: changedstatusjob['message'] = \ 'You can only thaw jobs with status: %s.'\ % ' or '.join(possible_thaw_states) status = returnvalues.CLIENT_ERROR changedstatusjobs.append(changedstatusjob) continue # job action is handled by changing the STATUS field, notifying the # job queue and making sure the server never submits jobs with status # FROZEN or CANCELED. # file is repickled to ensure newest information is used, job_dict # might be old if another script has modified the file. if not unpickle_and_change_status(filepath, new_state, logger): output_objects.append({ 'object_type': 'error_text', 'text': 'Job status could not be changed to %s!' % new_state }) status = returnvalues.SYSTEM_ERROR # Avoid key error and make sure grid_script gets expected number of # arguments if not job_dict.has_key('UNIQUE_RESOURCE_NAME'): job_dict['UNIQUE_RESOURCE_NAME'] = \ 'UNIQUE_RESOURCE_NAME_NOT_FOUND' if not job_dict.has_key('EXE'): job_dict['EXE'] = 'EXE_NAME_NOT_FOUND' # notify queue if not send_message_to_grid_script( 'JOBACTION ' + job_id + ' ' + job_dict['STATUS'] + ' ' + new_state + ' ' + job_dict['UNIQUE_RESOURCE_NAME'] + ' ' + job_dict['EXE'] + '\n', logger, configuration): output_objects.append({ 'object_type': 'error_text', 'text': '''Error sending message to grid_script, job may still be in the job queue.''' }) status = returnvalues.SYSTEM_ERROR continue changedstatusjob['newstatus'] = new_state changedstatusjobs.append(changedstatusjob) output_objects.append({ 'object_type': 'changedstatusjobs', 'changedstatusjobs': changedstatusjobs }) return (output_objects, status)
def run_transfer(configuration, client_id, transfer_dict): """Actual data transfer built from transfer_dict on behalf of client_id""" logger.debug('run transfer for %s: %s' % (client_id, blind_pw(transfer_dict))) transfer_id = transfer_dict['transfer_id'] action = transfer_dict['action'] protocol = transfer_dict['protocol'] status_dir = get_status_dir(configuration, client_id, transfer_id) cmd_map = get_cmd_map() if not protocol in cmd_map[action]: raise ValueError('unsupported protocol: %s' % protocol) client_dir = client_id_dir(client_id) makedirs_rec(status_dir, configuration) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep # TODO: we should refactor to move command extraction into one function command_pattern = cmd_map[action][protocol] target_helper_list = [] key_path = transfer_dict.get("key", "") if key_path: # Use key with given name from settings dir settings_base_dir = os.path.abspath( os.path.join(configuration.user_settings, client_dir)) + os.sep key_path = os.path.join(settings_base_dir, user_keys_dir, key_path.lstrip(os.sep)) # IMPORTANT: path must be expanded to abs for proper chrooting key_path = os.path.abspath(key_path) if not valid_user_path(configuration, key_path, settings_base_dir): logger.error('rejecting illegal directory traversal for %s (%s)' % (key_path, blind_pw(transfer_dict))) raise ValueError("user provided a key outside own settings!") rel_src_list = transfer_dict['src'] rel_dst = transfer_dict['dst'] compress = transfer_dict.get("compress", False) exclude = transfer_dict.get("exclude", []) if transfer_dict['action'] in ('import', ): logger.debug('setting abs dst for action %(action)s' % transfer_dict) src_path_list = transfer_dict['src'] dst_path = os.path.join(base_dir, rel_dst.lstrip(os.sep)) dst_path = os.path.abspath(dst_path) for src in rel_src_list: abs_dst = os.path.join(dst_path, src.lstrip(os.sep)) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dst = os.path.abspath(abs_dst) # Reject illegal directory traversal and hidden files if not valid_user_path(configuration, abs_dst, base_dir, True): logger.error( 'rejecting illegal directory traversal for %s (%s)' % (abs_dst, blind_pw(transfer_dict))) raise ValueError("user provided a destination outside home!") if src.endswith(os.sep): target_helper_list.append( (get_lftp_target(True, False, exclude), get_rsync_target(True, False, exclude, compress))) else: target_helper_list.append( (get_lftp_target(True, True, exclude), get_rsync_target(True, True, exclude, compress))) makedirs_rec(dst_path, configuration) elif transfer_dict['action'] in ('export', ): logger.debug('setting abs src for action %(action)s' % transfer_dict) dst_path = transfer_dict['dst'] src_path_list = [] for src in rel_src_list: src_path = os.path.join(base_dir, src.lstrip(os.sep)) # IMPORTANT: path must be expanded to abs for proper chrooting src_path = os.path.abspath(src_path) # Reject illegal directory traversal and hidden files if not valid_user_path(configuration, src_path, base_dir, True): logger.error( 'rejecting illegal directory traversal for %s (%s)' % (src, blind_pw(transfer_dict))) raise ValueError("user provided a source outside home!") src_path_list.append(src_path) if src.endswith(os.sep) or os.path.isdir(src): target_helper_list.append( (get_lftp_target(False, False, exclude), get_rsync_target(False, False, exclude, compress))) else: target_helper_list.append( (get_lftp_target(False, True, exclude), get_rsync_target(False, True, exclude, compress))) else: raise ValueError('unsupported action for %(transfer_id)s: %(action)s' % transfer_dict) run_dict = transfer_dict.copy() run_dict['log_path'] = os.path.join(status_dir, 'transfer.log') # Use private known hosts file for ssh transfers as explained above # NOTE: known_hosts containing '=' silently leads to rest getting ignored! # use /dev/null to skip host key verification completely for now. #run_dict['known_hosts'] = os.path.join(base_dir, '.ssh', 'known_hosts') run_dict['known_hosts'] = '/dev/null' # Make sure password is set to empty string as default run_dict['password'] = run_dict.get('password', '') # TODO: this is a bogus cert path for now - we don't support ssl certs run_dict['cert'] = run_dict.get('cert', '') # IMPORTANT: must be implicit proto or 'ftp://' (not ftps://) and similarly # webdav(s) must use explicit http(s) instead. In both cases we # replace protocol between cmd selection and lftp path expansion if run_dict['protocol'] == 'ftps': run_dict['orig_proto'] = run_dict['protocol'] run_dict['protocol'] = 'ftp' logger.info( 'force %(orig_proto)s to %(protocol)s for %(transfer_id)s' % run_dict) elif run_dict['protocol'].startswith('webdav'): run_dict['orig_proto'] = run_dict['protocol'] run_dict['protocol'] = run_dict['protocol'].replace('webdav', 'http') logger.info( 'force %(orig_proto)s to %(protocol)s for %(transfer_id)s' % run_dict) if key_path: rel_key = run_dict['key'] rel_cert = run_dict['cert'] run_dict['key'] = key_path run_dict['cert'] = key_path.replace(rel_key, rel_cert) run_dict['ssh_auth'] = get_ssh_auth(True, run_dict) run_dict['ssl_auth'] = get_ssl_auth(True, run_dict) else: # Extract encrypted password password_digest = run_dict.get('password_digest', '') if password_digest: _, _, _, payload = password_digest.split("$") unscrambled = unscramble_digest(configuration.site_digest_salt, payload) _, _, password = unscrambled.split(":") run_dict['password'] = password run_dict['ssh_auth'] = get_ssh_auth(False, run_dict) run_dict['ssl_auth'] = get_ssl_auth(False, run_dict) run_dict['rel_dst'] = rel_dst run_dict['dst'] = dst_path run_dict['lftp_buf_size'] = run_dict.get('lftp_buf_size', lftp_buffer_bytes) run_dict['lftp_sftp_block_size'] = run_dict.get('sftp_sftp_block_size', lftp_sftp_block_bytes) status = 0 for (src, rel_src, target_helper) in zip(src_path_list, rel_src_list, target_helper_list): (lftp_target, rsync_target) = target_helper logger.debug('setting up %(action)s for %(src)s' % run_dict) if run_dict['protocol'] == 'sftp' and not os.path.isabs(src): # NOTE: lftp interprets sftp://FQDN/SRC as absolute path /SRC # We force relative paths into user home with a tilde. # The resulting sftp://FQDN/~/SRC looks funky but works. run_dict['src'] = "~/%s" % src else: # All other paths are probably absolute or auto-chrooted anyway run_dict['src'] = src run_dict['rel_src'] = rel_src run_dict['lftp_args'] = ' '.join(lftp_target[0]) % run_dict run_dict['lftp_excludes'] = ' '.join(lftp_target[1]) # src and dst may actually be reversed for lftp, but for symmetry ... run_dict['lftp_src'] = lftp_target[2][0] % run_dict run_dict['lftp_dst'] = lftp_target[2][1] % run_dict run_dict['rsync_args'] = ' '.join(rsync_target[0]) % run_dict # Preserve excludes on list form for rsync, where it matters run_dict[RSYNC_EXCLUDES_LIST] = rsync_target[1] run_dict['rsync_src'] = rsync_target[2][0] % run_dict run_dict['rsync_dst'] = rsync_target[2][1] % run_dict blind_dict = blind_pw(run_dict) logger.debug('expanded vars to %s' % blind_dict) # NOTE: Make sure NOT to break rsync excludes on list form as they # won't work if concatenated to a single string in command_list! command_list, blind_list = [], [] for i in command_pattern: if i == RSYNC_EXCLUDES_LIST: command_list += run_dict[RSYNC_EXCLUDES_LIST] blind_list += run_dict[RSYNC_EXCLUDES_LIST] else: command_list.append(i % run_dict) blind_list.append(i % blind_dict) command_str = ' '.join(command_list) blind_str = ' '.join(blind_list) logger.info('run %s on behalf of %s' % (blind_str, client_id)) transfer_proc = subprocess_popen(command_list, stdout=subprocess_pipe, stderr=subprocess_pipe) # Save transfer_proc.pid for use in clean up during shutdown # in that way we can resume pretty smoothly in next run. sub_pid = transfer_proc.pid logger.info('%s %s running transfer process %s' % (client_id, transfer_id, sub_pid)) add_sub_pid(configuration, sub_pid_map, client_id, transfer_id, sub_pid) out, err = transfer_proc.communicate() exit_code = transfer_proc.wait() status |= exit_code del_sub_pid(configuration, sub_pid_map, client_id, transfer_id, sub_pid) logger.info('done running transfer %s: %s' % (transfer_id, blind_str)) logger.debug('raw output is: %s' % out) logger.debug('raw error is: %s' % err) logger.debug('result was %s' % exit_code) if not transfer_result(configuration, client_id, run_dict, exit_code, out.replace(base_dir, ''), err.replace(base_dir, '')): logger.error('writing transfer status for %s failed' % transfer_id) logger.debug('done handling transfers in %(transfer_id)s' % transfer_dict) transfer_dict['exit_code'] = status if status == 0: transfer_dict['status'] = 'DONE' else: transfer_dict['status'] = 'FAILED'
valid_job_id(pattern, min_length, max_length, extra_chars) def valid_user_path_name(safe_path, path, home_dir, allow_equal=False): """Wrap valid_user_path and valid_path name checks in one to check both destination dir and filename characters. Returns error using safe_path if validation fails. """ (status, msg) = (True, "") try: valid_path(path) except InputException, iex: status = False msg = "Invalid path! (%s: %s)" % (safe_path, iex) if not valid_user_path(path, home_dir, allow_equal): status = False msg = "Invalid path! (%s expands to illegal path)" % safe_path return (status, html_escape(msg)) def valid_email_address(addr): """Email check from http://www.secureprogramming.com/?action=view&feature=recipes&recipeid=1 """ rfc822_specials = '()<>@,;:\\"[]' # First we validate the name portion (name@domain) c = 0
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) job_ids = accepted['job_id'] action = accepted['action'][-1] src = accepted['src'] dst = accepted['dst'][-1] title_entry = find_entry(output_objects, 'title') title_entry['text'] = '%s live I/O' % configuration.short_title output_objects.append({'object_type': 'header', 'text' : 'Request live communication with jobs'}) if not action in valid_actions: output_objects.append({'object_type': 'error_text', 'text' : 'Invalid action "%s" (supported: %s)' % \ (action, ', '.join(valid_actions))}) return (output_objects, returnvalues.CLIENT_ERROR) if action in post_actions and not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) if not job_ids or action in interactive_actions: job_id = '' if job_ids: job_id = job_ids[-1] output_objects.append({'object_type': 'text', 'text' : ''' Fill in the live I/O details below to request communication with a running job. Job ID can be a full ID or a wild card pattern using "*" and "?" to match one or more of your job IDs. Use send output without source and destination paths to request upload of the default stdio files from the job on the resource to the associated job_output directory in your MiG home. Destination is a always handled as a directory path to put source files into. Source and destination paths are always taken relative to the job execution directory on the resource and your MiG home respectively. '''}) html = ''' <table class="liveio"> <tr> <td> <form method="post" action="liveio.py"> <table class="liveio"> <tr><td class=centertext> </td></tr> <tr><td> Action:<br /> <input type=radio name=action checked value="send" />send output <input type=radio name=action value="get" />get input </td></tr> <tr><td> Job ID:<br /> <input type=text size=60 name=job_id value="%s" /> </td></tr> <tr><td> Source path(s):<br /> <div id="srcfields"> <input type=text size=60 name=src value="" /><br /> </div> </td></tr> <tr><td> Destination path:<br /> <input type=text size=60 name=dst value="" /> </td></tr> <tr><td> <input type="submit" value="Send request" /> </td></tr> </table> </form> </td> <td> <script type="text/javascript"> fields = 1; max_fields = 64; function addInput() { if (fields < max_fields) { document.getElementById("srcfields").innerHTML += "<input type=text size=60 name=src value='' /><br />"; fields += 1; } else { alert("Maximum " + max_fields + " source fields allowed!"); document.form.add.disabled=true; } } </script> <form name="addsrcform"> <input type="button" onclick="addInput(); return false;" name="add" value="Add another source field" /> </form> </td> </tr> </table> ''' % job_id output_objects.append({'object_type': 'html_form', 'text' : html}) output_objects.append({'object_type': 'text', 'text': ''' Further live job control is avalable through your personal message queues. They provide a basic interface for centrally storing messages under your grid account and can be used to pass messages between jobs or for orchestrating jobs before and during execution. ''' }) output_objects.append({'object_type': 'link', 'destination': 'mqueue.py', 'text': 'Message queue interface'}) return (output_objects, returnvalues.OK) elif action in ['get', 'receive', 'input']: action = 'get' action_desc = 'will be downloaded to the job on the resource' elif action in ['put', 'send', 'output']: action = 'send' action_desc = 'will be uploaded from the job on the resource' else: output_objects.append({'object_type': 'error_text', 'text' : 'Invalid live io action: %s' % action}) return (output_objects, returnvalues.CLIENT_ERROR) output_objects.append({'object_type': 'text', 'text' : 'Requesting live I/O for %s' % ', '.join(job_ids)}) if action == 'get' and (not src or not dst): output_objects.append( {'object_type': 'error_text', 'text': 'src and dst parameters required for live input'}) return (output_objects, returnvalues.CLIENT_ERROR) # Automatic fall back to stdio files if output with no path provided if src: src_text = 'The files ' + ' '.join(src) else: src_text = 'The job stdio files' if dst: dst_text = 'the ' + dst + ' directory' else: dst_text = 'the corresponding job_output directory' # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep filelist = [] for job_id in job_ids: job_id = job_id.strip() # is job currently being executed? # Backward compatibility - all_jobs keyword should match all jobs if job_id == all_jobs: job_id = '*' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + job_id + '.mRSL') match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning("%s tried to %s restricted path %s ! (%s)" % \ (client_id, op_name, real_path, job_id)) continue # Insert valid job files in filelist for later treatment match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match.... if not match: output_objects.append( {'object_type': 'error_text', 'text' : '%s: You do not have any matching job IDs!' % job_id}) else: filelist += match for filepath in filelist: # Extract jo_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') job_dict = unpickle(filepath, logger) if not job_dict: status = returnvalues.CLIENT_ERROR output_objects.append( {'object_type': 'error_text', 'text' : ('You can only list status of your own jobs. ' 'Please verify that you submitted the mRSL file ' 'with job id "%s" (Could not unpickle mRSL file %s)' ) % (job_id, filepath)}) continue if job_dict['STATUS'] != 'EXECUTING': output_objects.append( {'object_type': 'text', 'text' : 'Job %s is not currently being executed! Job status: %s' % (job_id, job_dict['STATUS'])}) continue if job_dict['UNIQUE_RESOURCE_NAME'] == 'ARC': output_objects.append( {'object_type': 'text', 'text' : 'Job %s is submitted to ARC, details are not available!' % job_id }) continue last_live_update_dict = {} last_live_update_file = configuration.mig_system_files + os.sep\ + job_id + '.last_live_update' if os.path.isfile(last_live_update_file): last_live_update_dict_unpickled = \ unpickle(last_live_update_file, logger) if not last_live_update_dict_unpickled: output_objects.append({'object_type': 'error_text', 'text' : 'Could not unpickle %s - skipping request!' % last_live_update_file}) continue if not last_live_update_dict_unpickled.has_key( 'LAST_LIVE_UPDATE_REQUEST_TIMESTAMP'): output_objects.append( {'object_type': 'error_text', 'text': 'Could not find needed key in %s.' % last_live_update_file}) continue last_live_update_request = \ last_live_update_dict_unpickled['LAST_LIVE_UPDATE_REQUEST_TIMESTAMP' ] difference = datetime.datetime.now()- last_live_update_request try: min_delay = \ int(configuration.min_seconds_between_live_update_requests) except: min_delay = 30 if difference.seconds < min_delay: output_objects.append( {'object_type': 'error_text', 'text': ('Request not allowed, you must wait at least ' \ '%s seconds between live update requests!' ) % min_delay}) continue # save this request to file to avoid DoS from a client request loop. last_live_update_dict['LAST_LIVE_UPDATE_REQUEST_TIMESTAMP'] = \ datetime.datetime.now() pickle_ret = pickle(last_live_update_dict, last_live_update_file, logger) if not pickle_ret: output_objects.append( {'object_type': 'error_text', 'text' : 'Error saving live io request timestamp to last_live_update ' 'file, request not sent!'}) continue # # # ## job is being executed right now, send live io request to frontend # # # get resource_config, needed by scp_file_to_resource #(status, resource_config) = get_resource_configuration( # resource_home, unique_resource_name, logger) resource_config = job_dict['RESOURCE_CONFIG'] (status, exe) = get_resource_exe(resource_config, job_dict['EXE'], logger) if not status: output_objects.append( {'object_type': 'error_text', 'text' : 'Could not get exe configuration for job %s' % job_id}) continue local_file = '%s.%supdate' % (job_dict['LOCALJOBNAME'], action) if not os.path.exists(local_file): # create try: filehandle = open(local_file, 'w') filehandle.write('job_id ' + job_dict['JOB_ID'] + '\n') filehandle.write('localjobname ' + job_dict['LOCALJOBNAME'] + '\n') filehandle.write('execution_user ' + exe['execution_user'] + '\n') filehandle.write('execution_node ' + exe['execution_node'] + '\n') filehandle.write('execution_dir ' + exe['execution_dir'] + '\n') filehandle.write('target liveio\n') # Leave defaults src and dst to FE script if not provided if src: filehandle.write('source ' + ' '.join(src) + '\n') if dst: filehandle.write('destination ' + dst + '\n') # Backward compatible test for shared_fs - fall back to scp if exe.has_key('shared_fs') and exe['shared_fs']: filehandle.write('copy_command cp\n') filehandle.write('copy_frontend_prefix \n') filehandle.write('copy_execution_prefix \n') else: filehandle.write('copy_command scp -B\n') filehandle.write('copy_frontend_prefix ${frontend_user}@${frontend_node}:\n' ) filehandle.write('copy_execution_prefix ${execution_user}@${execution_node}:\n' ) filehandle.write('### END OF SCRIPT ###\n') filehandle.close() except Exception, exc: pass if not os.path.exists(local_file): output_objects.append( {'object_type': 'error_text', 'text' : '.%supdate file not available on %s server' % \ (action, configuration.short_title)}) continue scpstatus = copy_file_to_resource(local_file, '%s.%supdate' % (job_dict['LOCALJOBNAME'], action), resource_config, logger) if not scpstatus: output_objects.append( {'object_type': 'error_text', 'text' : 'Error sending request for live io to resource!'}) continue else: output_objects.append( {'object_type': 'text', 'text' : 'Request for live io was successfully sent to the resource!' }) output_objects.append( {'object_type': 'text', 'text' : '%s %s and should become available in %s in a minute.' % \ (src_text, action_desc, dst_text) }) if action == 'send': if not dst: target_path = '%s/%s/*' % (job_output_dir, job_id) else: target_path = dst output_objects.append({'object_type': 'link', 'destination' : 'ls.py?path=%s' % target_path, 'text': 'View uploaded files'}) else: output_objects.append({'object_type': 'link', 'destination' : 'ls.py?path=%s' % ';path='.join(src), 'text': 'View files for download'}) try: os.remove(local_file) except Exception, exc: pass
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) size = int(accepted['size'][-1]) pattern_list = accepted['path'] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) if size < 0: output_objects.append({ 'object_type': 'error_text', 'text': 'size must be non-negative' }) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') if verbose(flags): output_objects.append({ 'object_type': 'file', 'name': relative_path }) if not check_write_access(abs_path): logger.warning('%s called without write access: %s' % \ (op_name, abs_path)) output_objects.append( {'object_type': 'error_text', 'text': 'cannot truncate "%s": inside a read-only location!' % \ pattern}) status = returnvalues.CLIENT_ERROR continue try: fd = open(abs_path, 'r+') fd.truncate(size) fd.close() logger.info('%s %s %s done' % (op_name, abs_path, size)) except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False, op_menu=client_id) defaults = signature()[1] (validate_status, accepted) = validate_input(user_arguments_dict, defaults, output_objects, allow_rejects=False) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) queue = accepted['queue'][-1] action = accepted['action'][-1] iosessionid = accepted['iosessionid'][-1] msg = accepted['msg'][-1] msg_id = accepted['msg_id'][-1] # Web format for cert access and no header for SID access if client_id: output_objects.append({'object_type': 'header', 'text' : 'Message queue %s' % action}) else: output_objects.append({'object_type': 'start'}) # Always return at least a basic file_output entry file_entry = {'object_type': 'file_output', 'lines': [], 'wrap_binary': True, 'wrap_targets': ['lines']} if not action in valid_actions: output_objects.append({'object_type': 'error_text', 'text' : 'Invalid action "%s" (supported: %s)' % \ (action, ', '.join(valid_actions))}) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) if action in post_actions: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append( {'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Find user home from session or certificate if iosessionid: client_home = os.path.realpath(os.path.join(configuration.webserver_home, iosessionid)) client_dir = os.path.basename(client_home) elif client_id: client_dir = client_id_dir(client_id) else: output_objects.append({'object_type': 'error_text', 'text' : 'Either certificate or session ID is required' }) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if not os.path.isdir(base_dir): output_objects.append({'object_type': 'error_text', 'text' : 'No matching session or user home!'}) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) mqueue_base = os.path.join(base_dir, mqueue_prefix) + os.sep default_queue_dir = os.path.join(mqueue_base, default_mqueue) # Create mqueue base and default queue dir if missing if not os.path.exists(default_queue_dir): try: os.makedirs(default_queue_dir) except: pass # IMPORTANT: path must be expanded to abs for proper chrooting queue_path = os.path.abspath(os.path.join(mqueue_base, queue)) if not valid_user_path(configuration, queue_path, mqueue_base): output_objects.append({'object_type': 'error_text', 'text' : 'Invalid queue name: "%s"' % queue}) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) lock_path = os.path.join(mqueue_base, lock_name) lock_handle = open(lock_path, 'a') fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) status = returnvalues.OK if action == "interactive": form_method = 'post' csrf_limit = get_csrf_limit(configuration) fill_helpers = {'queue': queue, 'msg': msg, 'form_method': form_method, 'csrf_field': csrf_field, 'csrf_limit': csrf_limit, } target_op = 'mqueue' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token}) output_objects.append({'object_type': 'text', 'text': ''' Fill in the fields below to control and access your personal message queues. Jobs can receive from and send to the message queues during execution, and use them as a means of job inter-communication. Expect message queue operations to take several seconds on the resources, however. That is, use it for tasks like orchestrating long running jobs, and not for low latency communication. '''}) html = ''' <form name="mqueueform" method="%(form_method)s" action="%(target_op)s.py"> <table class="mqueue"> <tr><td class=centertext> </td></tr> <tr><td> Action:<br /> <input type="hidden" name="%(csrf_field)s" value="%(csrf_token)s" /> <input type=radio name=action value="create" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />create queue <input type=radio name=action checked value="send" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=false;" />send message to queue <input type=radio name=action value="receive" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />receive message from queue <input type=radio name=action value="remove" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />remove queue <input type=radio name=action value="listqueues" onclick="javascript: document.mqueueform.queue.disabled=true; document.mqueueform.msg.disabled=true;" />list queues <input type=radio name=action value="listmessages" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />list messages <input type=radio name=action value="show" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />show message </td></tr> <tr><td> Queue:<br /> <input class="fillwidth" type=text name=queue value="%(queue)s" /> </td></tr> <tr><td> <div id="msgfieldf"> <input class="fillwidth" type=text name=msg value="%(msg)s" /><br /> </div> </td></tr> <tr><td> <input type="submit" value="Apply" /> </td></tr> </table> </form> ''' % fill_helpers output_objects.append({'object_type': 'html_form', 'text' : html}) output_objects.append({'object_type': 'text', 'text': ''' Further live job control is avalable through the live I/O interface. They provide a basic interface for centrally managing input and output files for active jobs. ''' }) output_objects.append({'object_type': 'link', 'destination': 'liveio.py', 'text': 'Live I/O interface'}) return (output_objects, returnvalues.OK) elif action == 'create': try: os.mkdir(queue_path) output_objects.append({'object_type': 'text', 'text': 'New "%s" queue created' % queue}) except Exception, err: output_objects.append({'object_type': 'error_text', 'text' : 'Could not create "%s" queue: "%s"' % \ (queue, err)}) status = returnvalues.CLIENT_ERROR
filename_field, os.path.basename(rel_path), files_field, "dummy", csrf_field, csrf_token) uploaded.append(file_entry) logger.info('status done: %s' % ' '.join([i[0] for i in upload_files])) return (output_objects, status) elif action == 'move': # Move automatically takes place relative to dst_dir and current_dir for (rel_path, chunk_tuple) in upload_files: abs_src_path = os.path.abspath(os.path.join(base_dir, rel_path)) # IMPORTANT: path must be expanded to abs for proper chrooting dest_dir = os.path.abspath(os.path.join(base_dir, current_dir)) dest_path = os.path.join(dest_dir, os.path.basename(rel_path)) rel_dst = dest_path.replace(base_dir, '') if not valid_user_path(configuration, dest_path, base_dir, True): logger.error( '%s tried to %s move to restricted path %s ! (%s)' % (user_id, op_name, dest_path, current_dir)) output_objects.append({ 'object_type': 'error_text', 'text': "Invalid destination " "(%s expands to an illegal path)" % current_dir }) moved = False elif not check_write_access(dest_path, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, dest_path)) output_objects.append({
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = accepted['flags'] patterns = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND stats = [] for real_path in match: relative_path = real_path.replace(base_dir, '') try: (stat_status, stat) = stat_path(real_path, logger) if stat_status: if verbose(flags): stat['name'] = relative_path stat['object_type'] = 'stat' stats.append(stat) else: output_objects.append({'object_type': 'error_text', 'text': stat}) status = returnvalues.SYSTEM_ERROR except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'stats', 'stats' : stats})
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND submitstatuslist = [] for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') submitstatus = { 'object_type': 'submitstatus', 'name': relative_path } try: (job_status, newmsg, job_id) = new_job(abs_path, client_id, configuration, False, True) except Exception, exc: logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) job_status = False newmsg = "%s failed on '%s' (is it a valid mRSL file?)"\ % (op_name, relative_path) job_id = None if not job_status: submitstatus['status'] = False submitstatus['message'] = newmsg status = returnvalues.CLIENT_ERROR else: submitstatus['status'] = True submitstatus['job_id'] = job_id submitstatuslist.append(submitstatus) output_objects.append({ 'object_type': 'submitstatuslist', 'submitstatuslist': submitstatuslist })
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: logger.error("jobstatus input validation failed: %s" % accepted) return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) max_jobs = int(accepted['max_jobs'][-1]) order = 'unsorted ' if sorted(flags): order = 'sorted ' patterns = accepted['job_id'] project_names = accepted['project_name'] if len(project_names) > 0: for project_name in project_names: project_name_job_ids = \ get_job_ids_with_specified_project_name(client_id, project_name, configuration.mrsl_files_dir, logger) patterns.extend(project_name_job_ids) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep output_objects.append({'object_type': 'header', 'text' : '%s %s job status' % \ (configuration.short_title, order)}) if not patterns: output_objects.append({ 'object_type': 'error_text', 'text': 'No job_id specified!' }) return (output_objects, returnvalues.NO_SUCH_JOB_ID) if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) if not os.path.isdir(base_dir): output_objects.append( {'object_type': 'error_text', 'text' : ('You have not been created as a user on the %s server! ' \ 'Please contact the %s team.') % \ (configuration.short_title, configuration.short_title)}) return (output_objects, returnvalues.CLIENT_ERROR) filelist = [] for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to # avoid leaking information about file system layout while # allowing consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match.... if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % pattern }) status = returnvalues.CLIENT_ERROR else: filelist += match if sorted(flags): sort(filelist) if max_jobs > 0 and max_jobs < len(filelist): output_objects.append({ 'object_type': 'text', 'text': 'Only showing first %d of the %d matching jobs as requested' % (max_jobs, len(filelist)) }) filelist = filelist[:max_jobs] # Iterate through jobs and list details for each job_list = {'object_type': 'job_list', 'jobs': []} for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') job_dict = unpickle(filepath, logger) if not job_dict: status = returnvalues.CLIENT_ERROR output_objects.append( {'object_type': 'error_text', 'text' : 'No such job: %s (could not load mRSL file %s)' % \ (job_id, filepath)}) continue # Expand any job variables before use job_dict = expand_variables(job_dict) job_obj = {'object_type': 'job', 'job_id': job_id} job_obj['status'] = job_dict['STATUS'] time_fields = [ 'VERIFIED', 'VERIFIED_TIMESTAMP', 'RECEIVED_TIMESTAMP', 'QUEUED_TIMESTAMP', 'SCHEDULE_TIMESTAMP', 'EXECUTING_TIMESTAMP', 'FINISHED_TIMESTAMP', 'FAILED_TIMESTAMP', 'CANCELED_TIMESTAMP', ] for name in time_fields: if job_dict.has_key(name): # time objects cannot be marshalled, asctime if timestamp try: job_obj[name.lower()] = time.asctime(job_dict[name]) except Exception, exc: # not a time object, just add job_obj[name.lower()] = job_dict[name] ########################################### # ARC job status retrieval on demand: # But we should _not_ update the status in the mRSL files, since # other MiG code might rely on finding only valid "MiG" states. if configuration.arc_clusters and \ job_dict.get('UNIQUE_RESOURCE_NAME', 'unset') == 'ARC' \ and job_dict['STATUS'] == 'EXECUTING': try: home = os.path.join(configuration.user_home, client_dir) arcsession = arc.Ui(home) arcstatus = arcsession.jobStatus(job_dict['EXE']) job_obj['status'] = arcstatus['status'] except arc.ARCWrapperError, err: logger.error('Error retrieving ARC job status: %s' % \ err.what()) job_obj['status'] += '(Error: ' + err.what() + ')' except arc.NoProxyError, err: logger.error('While retrieving ARC job status: %s' % \ err.what()) job_obj['status'] += '(Error: ' + err.what() + ')'
if current_dir == upload_tmp_dir: file_entry["deleteType"] = "POST" file_entry["deleteUrl"] = del_url % \ (output_format, filename_field, os.path.basename(rel_path), files_field, "dummy") uploaded.append(file_entry) logger.info('status done: %s' % ' '.join([i[0] for i in upload_files])) return (output_objects, status) elif action == 'move': # Move automatically takes place relative to upload tmp dir for (rel_path, chunk_tuple) in upload_files: real_path = os.path.abspath(os.path.join(base_dir, rel_path)) dest_path = os.path.abspath(os.path.join( base_dir, current_dir, os.path.basename(rel_path))) if not valid_user_path(dest_path, base_dir, True): logger.error('%s tried to %s move to restricted path %s ! (%s)' % (client_id, op_name, dest_path, current_dir)) output_objects.append( {'object_type': 'error_text', 'text' : "Invalid destination (%s expands to an illegal path)" \ % current_dir}) moved = False else: try: move(real_path, dest_path) moved = True except Exception, exc: logger.error('could not move %s to %s: %s' % (real_path, dest_path, exc)) moved = False
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) path_list = accepted['path'] size_list = [int(size) for size in accepted['size']] title_entry = find_entry(output_objects, 'title') title_entry[ 'text'] = '%s Upload Progress Monitor' % configuration.short_title if not configuration.site_enable_griddk: output_objects.append({ 'object_type': 'text', 'text': '''Grid.dk features are disabled on this site. Please contact the site admins %s if you think they should be enabled. ''' % configuration.admin_email }) return (output_objects, returnvalues.OK) refresh_secs = 5 meta = '<meta http-equiv="refresh" content="%s" />' % refresh_secs title_entry['meta'] = meta # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep output_objects.append({'object_type': 'header', 'text': 'Upload progress'}) done_list = [False for _ in path_list] progress_items = [] index = -1 for path in path_list: index += 1 # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(os.path.join(base_dir, path)) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, path)) output_objects.append({ 'object_type': 'file_not_found', 'name': path }) status = returnvalues.FILE_NOT_FOUND continue if not os.path.isfile(abs_path): output_objects.append({ 'object_type': 'error_text', 'text': "no such upload: %s" % path }) status = returnvalues.CLIENT_ERROR continue relative_path = abs_path.replace(base_dir, '') try: logger.info('Checking size of upload %s' % abs_path) cur_size = os.path.getsize(abs_path) total_size = size_list[index] percent = round(100.0 * cur_size / total_size, 1) done_list[index] = (cur_size == total_size) progress_items.append({ 'object_type': 'progress', 'path': path, 'cur_size': cur_size, 'total_size': total_size, 'percent': percent, 'done': done_list[index], 'progress_type': 'upload' }) except Exception, exc: # Don't give away information about actual fs layout output_objects.append({ 'object_type': 'error_text', 'text': '%s upload could not be checked! (%s)' % (path, str(exc).replace(base_dir, '')) }) status = returnvalues.SYSTEM_ERROR continue
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] title_entry = find_entry(output_objects, 'title') label = "%s" % configuration.site_vgrid_label title_entry['text'] = "Add/Update %s Trigger" % label output_objects.append({ 'object_type': 'header', 'text': 'Add/Update %s Trigger' % label }) (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) # NOTE: strip leftmost slashes from all fields used in file paths to avoid # interference with os.path.join calls. Furthermore we strip and normalize # the path variable first to make sure it does not point outside the vgrid. # In practice any such directory traversal attempts will generally be moot # since the grid_events daemon only starts a listener for each top-level # vgrid and in there only reacts to events that match trigger rules from # that particular vgrid. Thus only subvgrid access to parent vgrids might # be a concern and still of limited consequence. # NOTE: merge multi args into one string and split again to get flat array rule_id = accepted['rule_id'][-1].strip() vgrid_name = accepted['vgrid_name'][-1].strip().lstrip(os.sep) path = os.path.normpath(accepted['path'][-1].strip()).lstrip(os.sep) changes = [i.strip() for i in ' '.join(accepted['changes']).split()] action = accepted['action'][-1].strip() arguments = [ i.strip() for i in shlex.split(' '.join(accepted['arguments'])) ] rate_limit = accepted['rate_limit'][-1].strip() settle_time = accepted['settle_time'][-1].strip() match_files = accepted['match_files'][-1].strip() == 'True' match_dirs = accepted['match_dirs'][-1].strip() == 'True' match_recursive = accepted['match_recursive'][-1].strip() == 'True' rank_str = accepted['rank'][-1] try: rank = int(rank_str) except ValueError: rank = None logger.debug("addvgridtrigger with args: %s" % user_arguments_dict) if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep # we just use a high res timestamp as automatic rule_id if rule_id == keyword_auto: rule_id = "%d" % (time.time() * 1E8) if action == keyword_auto: action = valid_trigger_actions[0] if any_state in changes: changes = valid_trigger_changes logger.info("addvgridtrigger %s" % vgrid_name) # Validity of user and vgrid names is checked in this init function so # no need to worry about illegal directory traversal through variables (ret_val, msg, ret_variables) = \ init_vgrid_script_add_rem(vgrid_name, client_id, rule_id, 'trigger', configuration) if not ret_val: output_objects.append({'object_type': 'error_text', 'text': msg}) return (output_objects, returnvalues.CLIENT_ERROR) elif msg: # In case of warnings, msg is non-empty while ret_val remains True output_objects.append({'object_type': 'warning', 'text': msg}) # if we get here user is either vgrid owner or allowed to add rule # don't add if already in vgrid or parent vgrid - but update if owner update_id = None if vgrid_is_trigger(vgrid_name, rule_id, configuration): if vgrid_is_trigger_owner(vgrid_name, rule_id, client_id, configuration): update_id = 'rule_id' else: output_objects.append({ 'object_type': 'error_text', 'text': '%s is already a trigger owned by somebody else in the %s' % (rule_id, label) }) return (output_objects, returnvalues.CLIENT_ERROR) # don't add if already in subvgrid (list_status, subvgrids) = vgrid_list_subvgrids(vgrid_name, configuration) if not list_status: output_objects.append({ 'object_type': 'error_text', 'text': 'Error getting list of sub%ss: %s' % (label, subvgrids) }) return (output_objects, returnvalues.SYSTEM_ERROR) for subvgrid in subvgrids: if vgrid_is_trigger(subvgrid, rule_id, configuration, recursive=False): output_objects.append({ 'object_type': 'error_text', 'text': '''%(rule_id)s is already in a sub-%(vgrid_label)s (%(subvgrid)s). Please remove the trigger from the sub-%(vgrid_label)s and try again''' % { 'rule_id': rule_id, 'subvgrid': subvgrid, 'vgrid_label': label } }) return (output_objects, returnvalues.CLIENT_ERROR) if not action in valid_trigger_actions: output_objects.append({ 'object_type': 'error_text', 'text': "invalid action value %s" % action }) return (output_objects, returnvalues.CLIENT_ERROR) if keyword_all in changes: changes = valid_trigger_changes for change in changes: if not change in valid_trigger_changes: output_objects.append({ 'object_type': 'error_text', 'text': "found invalid change value %s" % change }) return (output_objects, returnvalues.CLIENT_ERROR) # Check if we should load saved trigger for rank change or update rule_dict = None if rank is not None or update_id is not None: (load_status, all_triggers) = vgrid_triggers(vgrid_name, configuration) if not load_status: output_objects.append({ 'object_type': 'error_text', 'text': 'Failed to load triggers for %s: %s' % (vgrid_name, all_triggers) }) return (output_objects, returnvalues.SYSTEM_ERROR) for saved_dict in all_triggers: if saved_dict['rule_id'] == rule_id: rule_dict = saved_dict break if rule_dict is None: output_objects.append({ 'object_type': 'error_text', 'text': 'No such trigger %s for %s: %s' % (rule_id, vgrid_name, all_triggers) }) return (output_objects, returnvalues.CLIENT_ERROR) elif not path: # New trigger with missing path output_objects.append({ 'object_type': 'error_text', 'text': '''Either path or rank must be set.''' }) return (output_objects, returnvalues.CLIENT_ERROR) elif action == "submit" and not arguments: # New submit trigger with missing mrsl arguments output_objects.append({ 'object_type': 'error_text', 'text': '''Submit triggers must give a job description file path as argument.''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Handle create and update (i.e. new, update all or just refresh mRSL) if rank is None: # IMPORTANT: we save the job template contents to avoid potential abuse # Otherwise someone else in the VGrid could tamper with the template # and make the next trigger execute arbitrary code on behalf of the # rule owner. templates = [] # Merge current and saved values req_dict = { 'rule_id': rule_id, 'vgrid_name': vgrid_name, 'path': path, 'changes': changes, 'run_as': client_id, 'action': action, 'arguments': arguments, 'rate_limit': rate_limit, 'settle_time': settle_time, 'match_files': match_files, 'match_dirs': match_dirs, 'match_recursive': match_recursive, 'templates': templates } if rule_dict is None: rule_dict = req_dict else: for field in user_arguments_dict: if req_dict.has_key(field): rule_dict[field] = req_dict[field] # Now refresh template contents if rule_dict['action'] == "submit": for rel_path in rule_dict['arguments']: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(os.path.join(base_dir, rel_path)) try: if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning( '%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, rel_path)) raise ValueError('invalid submit path argument: %s' % rel_path) temp_fd = open(abs_path) templates.append(temp_fd.read()) temp_fd.close() except Exception, err: logger.error("read submit argument file failed: %s" % err) output_objects.append({ 'object_type': 'error_text', 'text': 'failed to read submit argument file "%s"' % rel_path }) return (output_objects, returnvalues.CLIENT_ERROR) # Save updated template contents here rule_dict['templates'] = templates
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) action = accepted['action'][-1] share_id = accepted['share_id'][-1] path = accepted['path'][-1] read_access = accepted['read_access'][-1].lower() in enabled_strings write_access = accepted['write_access'][-1].lower() in enabled_strings expire = accepted['expire'][-1] # Merge and split invite to make sure 'a@b, c@d' entries are handled invite_list = ','.join(accepted['invite']).split(',') invite_list = [i for i in invite_list if i] invite_msg = accepted['msg'] title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'Share Link' # jquery support for tablesorter and confirmation on delete/redo: # table initially sorted by 5, 4 reversed (active first and in growing age) table_spec = {'table_id': 'sharelinkstable', 'sort_order': '[[5,1],[4,1]]'} (add_import, add_init, add_ready) = man_base_js(configuration, [table_spec], {'width': 600}) title_entry['script']['advanced'] += add_import title_entry['script']['init'] += add_init title_entry['script']['ready'] += add_ready output_objects.append({ 'object_type': 'html_form', 'text': man_base_html(configuration) }) header_entry = {'object_type': 'header', 'text': 'Manage share links'} output_objects.append(header_entry) if not configuration.site_enable_sharelinks: output_objects.append({ 'object_type': 'text', 'text': ''' Share links are disabled on this site. Please contact the site admins %s if you think they should be enabled. ''' % configuration.admin_email }) return (output_objects, returnvalues.OK) logger.info('sharelink %s from %s' % (action, client_id)) logger.debug('sharelink from %s: %s' % (client_id, accepted)) if not action in valid_actions: output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid action "%s" (supported: %s)' % (action, ', '.join(valid_actions)) }) return (output_objects, returnvalues.CLIENT_ERROR) if action in post_actions: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) (load_status, share_map) = load_share_links(configuration, client_id) if not load_status: share_map = {} form_method = 'post' csrf_limit = get_csrf_limit(configuration) target_op = 'sharelink' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) if action in get_actions: if action == "show": # Table columns to skip skip_list = ['owner', 'single_file', 'expire'] sharelinks = [] for (saved_id, share_dict) in share_map.items(): share_item = build_sharelinkitem_object( configuration, share_dict) js_name = 'delete%s' % hexlify(saved_id) helper = html_post_helper( js_name, '%s.py' % target_op, { 'share_id': saved_id, 'action': 'delete', csrf_field: csrf_token }) output_objects.append({ 'object_type': 'html_form', 'text': helper }) share_item['delsharelink'] = { 'object_type': 'link', 'destination': "javascript: confirmDialog(%s, '%s');" % (js_name, 'Really remove %s?' % saved_id), 'class': 'removelink iconspace', 'title': 'Remove share link %s' % saved_id, 'text': '' } sharelinks.append(share_item) # Display share links and form to add new ones output_objects.append({ 'object_type': 'sectionheader', 'text': 'Share Links' }) output_objects.append({ 'object_type': 'table_pager', 'entry_name': 'share links', 'default_entries': default_pager_entries }) output_objects.append({ 'object_type': 'sharelinks', 'sharelinks': sharelinks, 'skip_list': skip_list }) output_objects.append({ 'object_type': 'html_form', 'text': '<br/>' }) output_objects.append({ 'object_type': 'sectionheader', 'text': 'Create Share Link' }) submit_button = '''<span> <input type=submit value="Create share link" /> </span>''' sharelink_html = create_share_link_form(configuration, client_id, 'html', submit_button, csrf_token) output_objects.append({ 'object_type': 'html_form', 'text': sharelink_html }) elif action == "edit": header_entry['text'] = 'Edit Share Link' share_dict = share_map.get(share_id, {}) if not share_dict: output_objects.append({ 'object_type': 'error_text', 'text': 'existing share link is required for edit' }) return (output_objects, returnvalues.CLIENT_ERROR) output_objects.append({ 'object_type': 'html_form', 'text': ''' <p> Here you can send invitations for your share link %(share_id)s to one or more comma-separated recipients. </p> ''' % share_dict }) sharelinks = [] share_item = build_sharelinkitem_object(configuration, share_dict) saved_id = share_item['share_id'] js_name = 'delete%s' % hexlify(saved_id) helper = html_post_helper(js_name, '%s.py' % target_op, { 'share_id': saved_id, 'action': 'delete', csrf_field: csrf_token }) output_objects.append({'object_type': 'html_form', 'text': helper}) # Hide link to self del share_item['editsharelink'] share_item['delsharelink'] = { 'object_type': 'link', 'destination': "javascript: confirmDialog(%s, '%s');" % (js_name, 'Really remove %s?' % saved_id), 'class': 'removelink iconspace', 'title': 'Remove share link %s' % saved_id, 'text': '' } sharelinks.append(share_item) output_objects.append({ 'object_type': 'sharelinks', 'sharelinks': sharelinks }) submit_button = '''<span> <input type=submit value="Send invitation(s)" /> </span>''' sharelink_html = invite_share_link_form(configuration, client_id, share_dict, 'html', submit_button, csrf_token) output_objects.append({ 'object_type': 'html_form', 'text': sharelink_html }) output_objects.append({ 'object_type': 'link', 'destination': 'sharelink.py', 'text': 'Return to share link overview' }) return (output_objects, returnvalues.OK) elif action in post_actions: share_dict = share_map.get(share_id, {}) if not share_dict and action != 'create': logger.warning('%s tried to %s missing or not owned link %s!' % (client_id, action, share_id)) output_objects.append({ 'object_type': 'error_text', 'text': '%s requires existing share link' % action }) return (output_objects, returnvalues.CLIENT_ERROR) share_path = share_dict.get('path', path) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep rel_share_path = share_path.lstrip(os.sep) # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(os.path.join(base_dir, rel_share_path)) relative_path = abs_path.replace(base_dir, '') real_path = os.path.realpath(abs_path) single_file = os.path.isfile(real_path) vgrid_name = in_vgrid_share(configuration, abs_path) if action == 'delete': header_entry['text'] = 'Delete Share Link' (save_status, _) = delete_share_link(share_id, client_id, configuration, share_map) if save_status and vgrid_name: logger.debug("del vgrid sharelink pointer %s" % share_id) (del_status, del_msg) = vgrid_remove_sharelinks(configuration, vgrid_name, [share_id], 'share_id') if not del_status: logger.error("del vgrid sharelink pointer %s failed: %s" % (share_id, del_msg)) return (False, share_map) desc = "delete" elif action == "update": header_entry['text'] = 'Update Share Link' # Try to point replies to client_id email client_email = extract_field(client_id, 'email') if invite_list: invites = share_dict.get('invites', []) + invite_list invites_uniq = list(set([i for i in invites if i])) invites_uniq.sort() share_dict['invites'] = invites_uniq auto_msg = invite_share_link_message(configuration, client_id, share_dict, 'html') msg = '\n'.join(invite_msg) # Now send request to all targets in turn threads = [] for target in invite_list: job_dict = { 'NOTIFY': [target.strip()], 'JOB_ID': 'NOJOBID', 'USER_CERT': client_id, 'EMAIL_SENDER': client_email } logger.debug('invite %s to %s' % (target, share_id)) threads.append( notify_user_thread( job_dict, [auto_msg, msg], 'INVITESHARE', logger, '', configuration, )) # Try finishing delivery but do not block forever on one message notify_done = [False for _ in threads] for _ in range(3): for i in range(len(invite_list)): if not notify_done[i]: logger.debug('check done %s' % invite_list[i]) notify = threads[i] notify.join(3) notify_done[i] = not notify.isAlive() notify_sent, notify_failed = [], [] for i in range(len(invite_list)): if notify_done[i]: notify_sent.append(invite_list[i]) else: notify_failed.append(invite_list[i]) logger.debug('notify sent %s, failed %s' % (notify_sent, notify_failed)) if notify_failed: output_objects.append({ 'object_type': 'html_form', 'text': ''' <p>Failed to send invitation to %s</p>''' % ', '.join(notify_failed) }) if notify_sent: output_objects.append({ 'object_type': 'html_form', 'text': '''<p>Invitation sent to %s</p> <textarea class="fillwidth padspace" rows="%d" readonly="readonly"> %s %s </textarea> ''' % (', '.join(notify_sent), (auto_msg + msg).count('\n') + 3, auto_msg, msg) }) if expire: share_dict['expire'] = expire (save_status, _) = update_share_link(share_dict, client_id, configuration, share_map) desc = "update" elif action == "create": header_entry['text'] = 'Create Share Link' if not read_access and not write_access: output_objects.append({ 'object_type': 'error_text', 'text': 'No access set - please select read, write or both' }) return (output_objects, returnvalues.CLIENT_ERROR) # NOTE: check path here as relative_path is empty for path='/' if not path: output_objects.append({ 'object_type': 'error_text', 'text': 'No path provided!' }) return (output_objects, returnvalues.CLIENT_ERROR) # We refuse sharing of entire home for security reasons elif not valid_user_path( configuration, abs_path, base_dir, allow_equal=False): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, action, abs_path, path)) output_objects.append({ 'object_type': 'error_text', 'text': '''Illegal path "%s": you can only share your own data, and not your entire home direcory.''' % path }) return (output_objects, returnvalues.CLIENT_ERROR) elif not os.path.exists(abs_path): output_objects.append({ 'object_type': 'error_text', 'text': 'Provided path "%s" does not exist!' % path }) return (output_objects, returnvalues.CLIENT_ERROR) # Refuse sharing of (mainly auth) dot dirs in root of user home elif real_path.startswith(os.path.join(base_dir, '.')): output_objects.append({ 'object_type': 'error_text', 'text': 'Provided path "%s" cannot be shared for security reasons' % path }) return (output_objects, returnvalues.CLIENT_ERROR) elif single_file and write_access: output_objects.append({ 'object_type': 'error_text', 'text': '''Individual files cannot be shared with write access - please share a directory with the file in it or only share with read access. ''' }) return (output_objects, returnvalues.CLIENT_ERROR) # We check if abs_path is in vgrid share, but do not worry about # private_base or public_base since they are only available to # owners, who can always share anyway. if vgrid_name is not None and \ not vgrid_is_owner(vgrid_name, client_id, configuration): # share is inside vgrid share so we must check that user is # permitted to create sharelinks there. (load_status, settings_dict) = vgrid_settings(vgrid_name, configuration, recursive=True, as_dict=True) if not load_status: # Probably owners just never saved settings, use defaults settings_dict = {'vgrid_name': vgrid_name} allowed = settings_dict.get('create_sharelink', keyword_owners) if allowed != keyword_members: output_objects.append({ 'object_type': 'error_text', 'text': '''The settings for the %(vgrid_name)s %(vgrid_label)s do not permit you to re-share %(vgrid_label)s shared folders. Please contact the %(vgrid_name)s owners if you think you should be allowed to do that. ''' % { 'vgrid_name': vgrid_name, 'vgrid_label': configuration.site_vgrid_label } }) return (output_objects, returnvalues.CLIENT_ERROR) access_list = [] if read_access: access_list.append('read') if write_access: access_list.append('write') share_mode = '-'.join((access_list + ['only'])[:2]) # TODO: more validity checks here if share_dict: desc = "update" else: desc = "create" # IMPORTANT: always use expanded path share_dict.update({ 'path': relative_path, 'access': access_list, 'expire': expire, 'invites': invite_list, 'single_file': single_file }) attempts = 1 generate_share_id = False if not share_id: attempts = 3 generate_share_id = True for i in range(attempts): if generate_share_id: share_id = generate_sharelink_id(configuration, share_mode) share_dict['share_id'] = share_id (save_status, save_msg) = create_share_link(share_dict, client_id, configuration, share_map) if save_status: logger.info('created sharelink: %s' % share_dict) break else: # ID Collision? logger.warning('could not create sharelink: %s' % save_msg) if save_status and vgrid_name: logger.debug("add vgrid sharelink pointer %s" % share_id) (add_status, add_msg) = vgrid_add_sharelinks(configuration, vgrid_name, [share_dict]) if not add_status: logger.error( "save vgrid sharelink pointer %s failed: %s " % (share_id, add_msg)) return (False, share_map) else: output_objects.append({ 'object_type': 'error_text', 'text': 'No such action %s' % action }) return (output_objects, returnvalues.CLIENT_ERROR) if not save_status: output_objects.append({ 'object_type': 'error_text', 'text': 'Error in %s share link %s: ' % (desc, share_id) + 'save updated share links failed!' }) return (output_objects, returnvalues.CLIENT_ERROR) output_objects.append({ 'object_type': 'text', 'text': '%sd share link %s on %s .' % (desc.title(), share_id, relative_path) }) if action in ['create', 'update']: sharelinks = [] share_item = build_sharelinkitem_object(configuration, share_dict) saved_id = share_item['share_id'] js_name = 'delete%s' % hexlify(saved_id) helper = html_post_helper(js_name, '%s.py' % target_op, { 'share_id': saved_id, 'action': 'delete', csrf_field: csrf_token }) output_objects.append({'object_type': 'html_form', 'text': helper}) share_item['delsharelink'] = { 'object_type': 'link', 'destination': "javascript: confirmDialog(%s, '%s');" % (js_name, 'Really remove %s?' % saved_id), 'class': 'removelink iconspace', 'title': 'Remove share link %s' % saved_id, 'text': '' } sharelinks.append(share_item) output_objects.append({ 'object_type': 'sharelinks', 'sharelinks': sharelinks }) if action == 'create': # NOTE: Leave editsharelink here for use in fileman overlay #del share_item['editsharelink'] output_objects.append({ 'object_type': 'html_form', 'text': '<br />' }) submit_button = '''<span> <input type=submit value="Send invitation(s)" /> </span>''' invite_html = invite_share_link_form(configuration, client_id, share_dict, 'html', submit_button, csrf_token) output_objects.append({ 'object_type': 'html_form', 'text': invite_html }) else: output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid share link action: %s' % action }) return (output_objects, returnvalues.CLIENT_ERROR) output_objects.append({'object_type': 'html_form', 'text': '<br />'}) output_objects.append({ 'object_type': 'link', 'destination': 'sharelink.py', 'text': 'Return to share link overview' }) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] current_dir = accepted['current_dir'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages # NB: Globbing disabled on purpose here unfiltered_match = [base_dir + pattern] match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append( {'object_type': 'error_text', 'text' : "%s: cannot remove directory '%s': Permission denied" % (op_name, pattern)}) status = returnvalues.CLIENT_ERROR for real_path in match: relative_path = real_path.replace(base_dir, '') if verbose(flags): output_objects.append({'object_type': 'file', 'name' : relative_path}) if not os.path.exists(real_path): output_objects.append({'object_type': 'file_not_found', 'name': relative_path}) continue try: if parents(flags): # Please note that 'rmdir -p X' should give error if X # doesn't exist contrary to 'mkdir -p X' not giving error # if X exists. os.removedirs(real_path) else: os.rmdir(real_path) except Exception, exc: output_objects.append({'object_type': 'error_text', 'text': "%s failed on '%s'" % (op_name, relative_path)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue
# outside home, which is checked later. path = os.path.abspath(path) if not path.startswith(home_path): logger.error("got path from %s outside user home: %s" % (client_ip, raw_path)) print INVALID_MARKER continue real_path = os.path.realpath(path) logger.debug("check path %s in home %s or chroot" % (path, home_path)) # Exact match to user home does not make sense as we expect a file # IMPORTANT: use path and not real_path here in order to test both if not valid_user_path(configuration, path, home_path, allow_equal=False, apache_scripts=True): logger.error("path from %s outside user chroot %s: %s (%s)" % (client_ip, home_path, raw_path, real_path)) print INVALID_MARKER continue logger.info("found valid user chroot path from %s: %s" % (client_ip, real_path)) print real_path # Throttle down a bit to yield time.sleep(0.01) except KeyboardInterrupt: keep_running = False
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) lines = int(accepted['lines'][-1]) pattern_list = accepted['path'] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') output_lines = [] # We search for the last 'lines' lines by beginning from the end of # the file and exponetially backtracking until the backtrack # contains at least 'lines' lines or we're back to the beginning of # the file. # At that point we skip any extra lines before printing. try: filedes = open(abs_path, 'r') # Go to end of file and backtrack backstep = 1 newlines = 0 filedes.seek(0, 2) length = filedes.tell() while backstep < length and newlines <= lines: backstep *= 2 if backstep > length: backstep = length filedes.seek(-backstep, 2) newlines = len(filedes.readlines()) if length: # Go back after reading to end of file filedes.seek(-backstep, 2) # Skip any extra lines caused by the exponential backtracking. # We could probably speed up convergence with binary search... while newlines > lines: dummy = filedes.readline() newlines -= 1 backstep = length - filedes.tell() # Now we're at the wanted spot - print rest of file for _ in range(newlines): output_lines.append(filedes.readline()) filedes.close() except Exception, exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = { 'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines'] } if verbose(flags): entry['path'] = relative_path output_objects.append(entry)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) pattern_list = accepted['path'] show_dest = accepted['with_dest'][0].lower() == 'true' listing = [] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK settings_dict = load_settings(client_id, configuration) javascript = '%s\n%s' % (select_all_javascript(), selected_file_actions_javascript()) title_entry = find_entry(output_objects, 'title') title_entry['text'] = '%s Files' % configuration.short_title title_entry['javascript'] = javascript output_objects.append({'object_type': 'header', 'text': '%s Files' % configuration.short_title }) location_pre_html = \ """ <div class='files'> <table class='files'> <tr class='title'><td class='centertext'> Working directory: </td></tr> <tr><td class='centertext'> """ output_objects.append({'object_type': 'html_form', 'text' : location_pre_html}) for pattern in pattern_list: links = [] links.append({'object_type': 'link', 'text': '%s HOME' % configuration.short_title, 'destination': 'ls.py?path=.'}) prefix = '' parts = pattern.split(os.sep) for i in parts: prefix = os.path.join(prefix, i) links.append({'object_type': 'link', 'text': i, 'destination': 'ls.py?path=%s' % prefix}) output_objects.append({'object_type': 'multilinkline', 'links' : links}) location_post_html = """ </td></tr> </table> </div> <br /> """ output_objects.append({'object_type': 'html_form', 'text' : location_post_html}) more_html = \ """ <div class='files'> <form method='post' name='fileform' onSubmit='return selectedFilesAction();'> <table class='files'> <tr class='title'><td class='centertext' colspan=2> Advanced file actions </td></tr> <tr><td> Action on paths selected below (please hold mouse cursor over button for a description): </td> <td class='centertext'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='v' /> <input type='submit' title='Show concatenated contents (cat)' onClick='document.pressed=this.value' value='cat' /> <input type='submit' onClick='document.pressed=this.value' value='head' title='Show first lines (head)' /> <input type='submit' onClick='document.pressed=this.value' value='tail' title='Show last lines (tail)' /> <input type='submit' onClick='document.pressed=this.value' value='wc' title='Count lines/words/chars (wc)' /> <input type='submit' onClick='document.pressed=this.value' value='stat' title='Show details (stat)' /> <input type='submit' onClick='document.pressed=this.value' value='touch' title='Update timestamp (touch)' /> <input type='submit' onClick='document.pressed=this.value' value='truncate' title='truncate! (truncate)' /> <input type='submit' onClick='document.pressed=this.value' value='rm' title='delete! (rm)' /> <input type='submit' onClick='document.pressed=this.value' value='rmdir' title='Remove directory (rmdir)' /> <input type='submit' onClick='document.pressed=this.value' value='submit' title='Submit file (submit)' /> </td></tr> </table> </form> </div> """ output_objects.append({'object_type': 'html_form', 'text' : more_html}) dir_listings = [] output_objects.append({ 'object_type': 'dir_listings', 'dir_listings': dir_listings, 'flags': flags, 'show_dest': show_dest, }) first_match = None for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: real_path = os.path.abspath(server_path) if not valid_user_path(real_path, base_dir, True): logger.warning('%s tried to %s restricted path %s! (%s)' % (client_id, op_name, real_path, pattern)) continue match.append(real_path) if not first_match: first_match = real_path # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for real_path in match: if real_path + os.sep == base_dir: relative_path = '.' else: relative_path = real_path.replace(base_dir, '') entries = [] dir_listing = { 'object_type': 'dir_listing', 'relative_path': relative_path, 'entries': entries, 'flags': flags, } dest = '' if show_dest: if os.path.isfile(real_path): dest = os.path.basename(real_path) elif recursive(flags): # references to '.' or similar are stripped by abspath if real_path + os.sep == base_dir: dest = '' else: # dest = os.path.dirname(real_path).replace(base_dir, "") dest = os.path.basename(real_path) + os.sep handle_expand( output_objects, entries, base_dir, real_path, flags, dest, 0, show_dest, ) dir_listings.append(dir_listing) output_objects.append({'object_type': 'html_form', 'text' : """ <div class='files'> <table class='files'> <tr class='title'><td class='centertext'> Filter paths (wildcards like * and ? are allowed) <form method='post' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%s' /> <input type='text' name='path' value='' /> <input type='submit' value='Filter' /> </form> </td></tr> </table> </div> """ % flags}) # Short/long format buttons htmlform = \ """<table class='files'> <tr class='title'><td class='centertext' colspan=4> File view options </td></tr> <tr><td colspan=4><br /></td></tr> <tr class='title'><td>Parameter</td><td>Setting</td><td>Enable</td><td>Disable</td></tr> <tr><td>Long format</td><td> %s</td><td>"""\ % long_list(flags)\ + """ <form method='post' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%s' />"""\ % (flags + 'l') for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />"\ % entry htmlform += \ """ <input type='submit' value='On' /><br /> </form> </td><td> <form method='post' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%s' />"""\ % flags.replace('l', '') for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />"\ % entry htmlform += \ """ <input type='submit' value='Off' /><br /> </form> </td></tr> """ # Recursive output htmlform += \ """ <!-- Non-/recursive list buttons --> <tr><td>Recursion</td><td> %s</td><td>"""\ % recursive(flags) htmlform += \ """ <form method='post' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%s' />"""\ % (flags + 'r') for entry in pattern_list: htmlform += " <input type='hidden' name='path' value='%s' />"\ % entry htmlform += \ """ <input type='submit' value='On' /><br /> </form> </td><td> <form method='post' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%s' />"""\ % flags.replace('r', '') for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />"\ % entry htmlform += \ """ <input type='submit' value='Off' /><br /> </form> </td></tr> """ htmlform += \ """ <!-- Show dot files buttons --> <tr><td>Show hidden files</td><td> %s</td><td>"""\ % all(flags) htmlform += \ """ <form method='post' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%s' />"""\ % (flags + 'a') for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />"\ % entry htmlform += \ """ <input type='submit' value='On' /><br /> </form> </td><td> <form method='post' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%s' />"""\ % flags.replace('a', '') for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />"\ % entry htmlform += \ """ <input type='submit' value='Off' /><br /> </form> </td></tr> </table> """ # show flag buttons after contents to avoid output_objects.append({'object_type': 'html_form', 'text' : htmlform}) # create upload file form if first_match: # use first match for current directory # Note that base_dir contains an ending slash if os.path.isdir(first_match): dir_path = first_match else: dir_path = os.path.dirname(first_match) if dir_path + os.sep == base_dir: relative_dir = '.' else: relative_dir = dir_path.replace(base_dir, '') output_objects.append({'object_type': 'html_form', 'text' : """ <br /> <table class='files'> <tr class='title'><td class='centertext' colspan=2> Edit file </td><td><br /></td></tr> <tr><td> Fill in the path of a file to edit and press 'edit' to open that file in the<br /> online file editor. Alternatively a file can be selected for editing through<br /> the listing of personal files. </td><td colspan=2 class='righttext'> <form name='editor' method='post' action='editor.py'> <input type='hidden' name='output_format' value='html' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> <input type='text' name='path' size=50 value='' /> <input type='submit' value='edit' /> </form> </td></tr> </table> <br /> <table class='files'> <tr class='title'><td class='centertext' colspan=4> Create directory </td></tr> <tr><td> Name of new directory to be created in current directory (%(dest_dir)s) </td><td class='righttext' colspan=3> <form action='mkdir.py' method=post> <input name='path' size=50 /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> <input type='submit' value='Create' name='mkdirbutton' /> </form> </td></tr> </table> <br /> <form enctype='multipart/form-data' action='textarea.py' method='post'> <table class='files'> <tr class='title'><td class='centertext' colspan=4> Upload file </td></tr> <tr><td colspan=4> Upload file to current directory (%(dest_dir)s) </td></tr> <tr><td colspan=2> Extract package files (.zip, .tar.gz, .tar.bz2) </td><td colspan=2> <input type='checkbox' name='extract_0' /> </td></tr> <tr><td colspan=2> Submit mRSL files (also .mRSL files included in packages) </td><td colspan=2> <input type='checkbox' name='submitmrsl_0' checked /> </td></tr> <tr><td> File to upload </td><td class='righttext' colspan=3> <input name='fileupload_0_0_0' type='file' /> </td></tr> <tr><td> Optional remote filename (extra useful in windows) </td><td class='righttext' colspan=3> <input name='default_remotefilename_0' type='hidden' value='%(dest_dir)s' /> <input name='remotefilename_0' type='text' size='50' value='%(dest_dir)s' /> <input type='submit' value='Upload' name='sendfile' /> </td></tr> </table> </form> """ % {'dest_dir': relative_dir + os.sep}}) return (output_objects, status)