def is_owner( client_id, unique_config_name, config_home, logger, ): """Check that client_id is listed in pickled owners file""" config_path = os.path.abspath(os.path.join(config_home, unique_config_name, 'owners')) # Check validity of unique_config_name # Automatic configuration extraction configuration = None if not valid_user_path(configuration, config_path, config_home): # Extract caller information from traceback import format_stack caller = ''.join(format_stack()[:-1]).strip() logger.warning("""is_owner caught possible illegal directory traversal attempt by client: '%s' unique name: '%s' caller: %s""" % (client_id, unique_config_name, caller)) return False return is_item_in_pickled_list(config_path, client_id, logger)
# Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = fileinfo_dict['base_path'] = fileinfo_dict['base_path'] + os.sep # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages path = fileinfo_dict['path'] unfiltered_match = [base_dir + os.sep + fileinfo_dict['path']] match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to access restricted path %s ! (%s)' % (client_id, abs_path, path)) o.out('Access to %s is prohibited!' % fileinfo_dict['path']) o.reply_and_exit(o.CLIENT_ERROR) if action == 'GET': status = get(o, fileinfo_dict) # if status, we have already written to the client, see get method. if not status: o.reply_and_exit(o.ERROR) elif action == 'PUT': status = put(o, fileinfo_dict)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: path cannot use wildcards here typecheck_overrides={}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) path = accepted['path'][-1] chosen_newline = accepted['newline'][-1] submitjob = accepted['submitjob'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_jobs and submitjob: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep # HTML spec dictates newlines in forms to be MS style (\r\n) # rather than un*x style (\n): change if requested. form_newline = '\r\n' allowed_newline = {'unix': '\n', 'mac': '\r', 'windows': '\r\n'} output_objects.append({ 'object_type': 'header', 'text': 'Saving changes to edited file' }) if not chosen_newline in allowed_newline.keys(): output_objects.append({ 'object_type': 'error_text', 'text': 'Unsupported newline style supplied: %s (must be one of %s)' % (chosen_newline, ', '.join(allowed_newline.keys())) }) return (output_objects, returnvalues.CLIENT_ERROR) saved_newline = allowed_newline[chosen_newline] # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing consistent # error messages abs_path = '' unfiltered_match = glob.glob(base_dir + path) for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, path)) output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % path }) return (output_objects, returnvalues.CLIENT_ERROR) if abs_path == '': # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(os.path.join(base_dir, path.lstrip(os.sep))) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, path)) output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % path }) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_path, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_path)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot edit "%s": inside a read-only location!' % path }) status = returnvalues.CLIENT_ERROR return (output_objects, returnvalues.CLIENT_ERROR) (owner, time_left) = acquire_edit_lock(abs_path, client_id) if owner != client_id: output_objects.append({ 'object_type': 'error_text', 'text': "You don't have the lock for %s!" % path }) return (output_objects, returnvalues.CLIENT_ERROR) try: fh = open(abs_path, 'w+') fh.write(user_arguments_dict['editarea'][0].replace( form_newline, saved_newline)) fh.close() # everything ok output_objects.append({ 'object_type': 'text', 'text': 'Saved changes to %s.' % path }) logger.info('saved changes to %s' % path) release_edit_lock(abs_path, client_id) except Exception as exc: # Don't give away information about actual fs layout output_objects.append({ 'object_type': 'error_text', 'text': '%s could not be written! (%s)' % (path, str(exc).replace(base_dir, '')) }) return (output_objects, returnvalues.SYSTEM_ERROR) if submitjob: output_objects.append({ 'object_type': 'text', 'text': 'Submitting saved file to parser' }) submitstatus = {'object_type': 'submitstatus', 'name': path} (new_job_status, msg, job_id) = new_job(abs_path, client_id, configuration, False, True) if not new_job_status: submitstatus['status'] = False submitstatus['message'] = msg else: submitstatus['status'] = True submitstatus['job_id'] = job_id output_objects.append({ 'object_type': 'submitstatuslist', 'submitstatuslist': [submitstatus] }) output_objects.append({ 'object_type': 'link', 'destination': 'javascript:history.back()', 'class': 'backlink iconspace', 'title': 'Go back to previous page', 'text': 'Back to previous page' }) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: path can use wildcards typecheck_overrides={'path': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] search = accepted['pattern'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag)}) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') output_lines = [] try: matching = pattern_match_file(search, abs_path) for line in matching: output_lines.append(line) except Exception as exc: output_objects.append({'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = {'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines']} if verbose(flags): entry['path'] = relative_path output_objects.append(entry) return (output_objects, status)
# Single file sharelinks use direct link to file. If so we # manually expand to direct target. Otherwise we only replace # that prefix of path to translate it to a sharelink dir path. if is_file: logger.debug("found single file link: %s" % path) path = link_target else: logger.debug("found directory link: %s" % path) path = path.replace(full_prefix, link_target, 1) real_path = os.path.realpath(path) logger.info("check path from %s in base %s or chroot: %s" % (client_ip, base_path, path)) # Exact match to sid dir does not make sense as we expect a file # IMPORTANT: use path and not real_path here in order to test both if not valid_user_path(configuration, path, base_path, allow_equal=is_file, apache_scripts=True): logger.error("request from %s is outside sid chroot %s: %s (%s)" % (client_ip, base_path, raw_path, real_path)) print(INVALID_MARKER) continue logger.info("found valid sid chroot path from %s: %s" % (client_ip, real_path)) print(real_path) # Throttle down a bit to yield time.sleep(0.01) except KeyboardInterrupt: keep_running = False except Exception as exc: logger.error("unexpected exception: %s" % exc)
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False, op_menu=client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input( user_arguments_dict, defaults, output_objects, allow_rejects=False, # NOTE: path cannot use wildcards here typecheck_overrides={}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] current_dir = accepted['current_dir'][-1] share_id = accepted['share_id'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Either authenticated user client_id set or sharelink ID if client_id: user_id = client_id target_dir = client_id_dir(client_id) base_dir = configuration.user_home id_query = '' page_title = 'Create User Directory' userstyle = True widgets = True elif share_id: try: (share_mode, _) = extract_mode_id(configuration, share_id) except ValueError as err: logger.error('%s called with invalid share_id %s: %s' % (op_name, share_id, err)) output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid sharelink ID: %s' % share_id }) return (output_objects, returnvalues.CLIENT_ERROR) # TODO: load and check sharelink pickle (currently requires client_id) user_id = 'anonymous user through share ID %s' % share_id if share_mode == 'read-only': logger.error('%s called without write access: %s' % (op_name, accepted)) output_objects.append({ 'object_type': 'error_text', 'text': 'No write access!' }) return (output_objects, returnvalues.CLIENT_ERROR) target_dir = os.path.join(share_mode, share_id) base_dir = configuration.sharelink_home id_query = '?share_id=%s' % share_id page_title = 'Create Shared Directory' userstyle = False widgets = False else: logger.error('%s called without proper auth: %s' % (op_name, accepted)) output_objects.append({ 'object_type': 'error_text', 'text': 'Authentication is missing!' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(base_dir, target_dir)) + os.sep title_entry = find_entry(output_objects, 'title') title_entry['text'] = page_title title_entry['skipwidgets'] = not widgets title_entry['skipuserstyle'] = not userstyle output_objects.append({'object_type': 'header', 'text': page_title}) # Input validation assures target_dir can't escape base_dir if not os.path.isdir(base_dir): output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid client/sharelink id!' }) return (output_objects, returnvalues.CLIENT_ERROR) if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages # NB: Globbing disabled on purpose here unfiltered_match = [base_dir + os.sep + current_dir + os.sep + pattern] match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warn('%s tried to %s %s restricted path! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'error_text', 'text': "%s: cannot create directory '%s': Permission denied" % (op_name, pattern) }) status = returnvalues.CLIENT_ERROR for abs_path in match: relative_path = abs_path.replace(base_dir, '') if verbose(flags): output_objects.append({ 'object_type': 'file', 'name': relative_path }) if not parents(flags) and os.path.exists(abs_path): output_objects.append({ 'object_type': 'error_text', 'text': '%s: path exist!' % pattern }) status = returnvalues.CLIENT_ERROR continue if not check_write_access(abs_path, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_path)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot create "%s": inside a read-only location!' % pattern }) status = returnvalues.CLIENT_ERROR continue try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'created', [relative_path]) if parents(flags): if not os.path.isdir(abs_path): os.makedirs(abs_path) else: os.mkdir(abs_path) logger.info('%s %s done' % (op_name, abs_path)) except Exception as exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'created', [relative_path], failed=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s' failed!" % (op_name, relative_path) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({ 'object_type': 'text', 'text': "created directory %s" % (relative_path) }) if id_query: open_query = "%s;current_dir=%s" % (id_query, relative_path) else: open_query = "?current_dir=%s" % relative_path output_objects.append({ 'object_type': 'link', 'destination': 'ls.py%s' % open_query, 'text': 'Open %s' % relative_path }) output_objects.append({'object_type': 'text', 'text': ''}) output_objects.append({ 'object_type': 'link', 'destination': 'ls.py%s' % id_query, 'text': 'Return to files overview' }) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) patterns = accepted['job_id'] action = accepted['action'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) if not action in valid_actions.keys(): output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid job action "%s" (only %s supported)' % (action, ', '.join(valid_actions.keys())) }) return (output_objects, returnvalues.CLIENT_ERROR) new_state = valid_actions[action] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep status = returnvalues.OK filelist = [] for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.error( '%s tried to use %s %s outside own home! (pattern %s)' % (client_id, op_name, abs_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % pattern }) status = returnvalues.CLIENT_ERROR else: filelist += match # job state change is hard on the server, limit if len(filelist) > 500: output_objects.append({ 'object_type': 'error_text', 'text': 'Too many matching jobs (%s)!' % len(filelist) }) return (output_objects, returnvalues.CLIENT_ERROR) changedstatusjobs = [] for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') changedstatusjob = { 'object_type': 'changedstatusjob', 'job_id': job_id } job_dict = unpickle(filepath, logger) if not job_dict: changedstatusjob['message'] = '''The file containing the information for job id %s could not be opened! You can only %s your own jobs!''' % (job_id, action) changedstatusjobs.append(changedstatusjob) status = returnvalues.CLIENT_ERROR continue changedstatusjob['oldstatus'] = job_dict['STATUS'] # Is the job status compatible with action? possible_cancel_states = [ 'PARSE', 'QUEUED', 'RETRY', 'EXECUTING', 'FROZEN' ] if action == 'cancel' and \ not job_dict['STATUS'] in possible_cancel_states: changedstatusjob['message'] = \ 'You can only cancel jobs with status: %s.'\ % ' or '.join(possible_cancel_states) status = returnvalues.CLIENT_ERROR changedstatusjobs.append(changedstatusjob) continue possible_freeze_states = ['QUEUED', 'RETRY'] if action == 'freeze' and \ not job_dict['STATUS'] in possible_freeze_states: changedstatusjob['message'] = \ 'You can only freeze jobs with status: %s.'\ % ' or '.join(possible_freeze_states) status = returnvalues.CLIENT_ERROR changedstatusjobs.append(changedstatusjob) continue possible_thaw_states = ['FROZEN'] if action == 'thaw' and \ not job_dict['STATUS'] in possible_thaw_states: changedstatusjob['message'] = \ 'You can only thaw jobs with status: %s.'\ % ' or '.join(possible_thaw_states) status = returnvalues.CLIENT_ERROR changedstatusjobs.append(changedstatusjob) continue # job action is handled by changing the STATUS field, notifying the # job queue and making sure the server never submits jobs with status # FROZEN or CANCELED. # file is repickled to ensure newest information is used, job_dict # might be old if another script has modified the file. if not unpickle_and_change_status(filepath, new_state, logger): output_objects.append({ 'object_type': 'error_text', 'text': 'Job status could not be changed to %s!' % new_state }) status = returnvalues.SYSTEM_ERROR # Avoid key error and make sure grid_script gets expected number of # arguments if 'UNIQUE_RESOURCE_NAME' not in job_dict: job_dict['UNIQUE_RESOURCE_NAME'] = \ 'UNIQUE_RESOURCE_NAME_NOT_FOUND' if 'EXE' not in job_dict: job_dict['EXE'] = 'EXE_NAME_NOT_FOUND' # notify queue if not send_message_to_grid_script( 'JOBACTION ' + job_id + ' ' + job_dict['STATUS'] + ' ' + new_state + ' ' + job_dict['UNIQUE_RESOURCE_NAME'] + ' ' + job_dict['EXE'] + '\n', logger, configuration): output_objects.append({ 'object_type': 'error_text', 'text': '''Error sending message to grid_script, job may still be in the job queue.''' }) status = returnvalues.SYSTEM_ERROR continue changedstatusjob['newstatus'] = new_state changedstatusjobs.append(changedstatusjob) output_objects.append({ 'object_type': 'changedstatusjobs', 'changedstatusjobs': changedstatusjobs }) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) defaults = signature()[1] output_objects.append({ 'object_type': 'header', 'text': 'Reject Resource Request' }) (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) unique_resource_name = accepted['unique_resource_name'][-1].strip() request_name = unhexlify(accepted['request_name'][-1]) if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not is_owner(client_id, unique_resource_name, configuration.resource_home, logger): output_objects.append({ 'object_type': 'error_text', 'text': 'You must be an owner of %s to reject requests!' % unique_resource_name }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # resource dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.resource_home, unique_resource_name)) + os.sep # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(os.path.join(base_dir, request_name)) if not valid_user_path( configuration, abs_path, base_dir, allow_equal=False): logger.warning('%s tried to access restricted path %s ! (%s)' % \ (client_id, abs_path, request_name)) output_objects.append({ 'object_type': 'error_text', 'text': '''Illegal request name "%s": you can only reject requests to your own resources.''' % request_name }) return (output_objects, returnvalues.CLIENT_ERROR) if request_name: request_dir = os.path.join(configuration.resource_home, unique_resource_name) req = load_access_request(configuration, request_dir, request_name) if not req or not delete_access_request(configuration, request_dir, request_name): logger.error("failed to delete owner request for %s in %s" % \ (unique_resource_name, request_name)) output_objects.append({ 'object_type': 'error_text', 'text': 'Failed to remove saved resource request for %s in %s!'\ % (unique_resource_name, request_name)}) return (output_objects, returnvalues.CLIENT_ERROR) output_objects.append({ 'object_type': 'text', 'text': ''' Deleted %(request_type)s access request to %(target)s for %(entity)s . ''' % req }) form_method = 'post' csrf_limit = get_csrf_limit(configuration) fill_helpers = { 'protocol': any_protocol, 'unique_resource_name': unique_resource_name, 'form_method': form_method, 'csrf_field': csrf_field, 'csrf_limit': csrf_limit } fill_helpers.update(req) target_op = 'sendrequestaction' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token}) output_objects.append({ 'object_type': 'html_form', 'text': """ <p> You can use the reply form below if you want to additionally send an explanation for rejecting the request. </p> <form method='%(form_method)s' action='%(target_op)s.py'> <input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' /> <input type=hidden name=request_type value='resourcereject' /> <input type=hidden name=unique_resource_name value='%(target)s' /> <input type=hidden name=cert_id value='%(entity)s' /> <input type=hidden name=protocol value='%(protocol)s' /> <table> <tr> <td class='title'>Optional reject message to requestor(s)</td> </tr><tr> <td><textarea name=request_text cols=72 rows=10> We have decided to reject your %(request_type)s request to our %(target)s resource. Regards, the %(target)s resource owners </textarea></td> </tr> <tr> <td><input type='submit' value='Inform requestor(s)' /></td> </tr> </table> </form> <br /> """ % fill_helpers }) output_objects.append({ 'object_type': 'link', 'destination': 'resadmin.py?unique_resource_name=%s' % unique_resource_name, 'text': 'Back to administration for %s' % unique_resource_name }) return (output_objects, returnvalues.OK)
# outside home, which is checked later. path = os.path.abspath(path) if not path.startswith(home_path): logger.error("got path from %s outside user home: %s" % (client_ip, raw_path)) print(INVALID_MARKER) continue real_path = os.path.realpath(path) logger.debug("check path %s in home %s or chroot" % (path, home_path)) # Exact match to user home does not make sense as we expect a file # IMPORTANT: use path and not real_path here in order to test both if not valid_user_path(configuration, path, home_path, allow_equal=False, apache_scripts=True): logger.error("path from %s outside user chroot %s: %s (%s)" % (client_ip, home_path, raw_path, real_path)) print(INVALID_MARKER) continue elif not check_account_accessible(configuration, user_id, 'https'): # Only warn to avoid excessive noise from scanners logger.warning( "path from %s in inaccessible %s account: %s (%s)" % (client_ip, user_id, raw_path, real_path)) print(INVALID_MARKER) continue logger.info("found valid user chroot path from %s: %s" %
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: src and dst can use wildcards here typecheck_overrides={'src': valid_path_pattern, 'dst': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) src_list = accepted['src'] dst = accepted['dst'][-1] iosessionid = accepted['iosessionid'][-1] share_id = accepted['share_id'][-1] freeze_id = accepted['freeze_id'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append( {'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep # Special handling if used from a job (no client_id but iosessionid) if not client_id and iosessionid: base_dir = os.path.realpath(configuration.webserver_home + os.sep + iosessionid) + os.sep # Use selected base as source and destination dir by default src_base = dst_base = base_dir # Sharelink import if share_id is given - change to sharelink as src base if share_id: try: (share_mode, _) = extract_mode_id(configuration, share_id) except ValueError as err: logger.error('%s called with invalid share_id %s: %s' % (op_name, share_id, err)) output_objects.append( {'object_type': 'error_text', 'text': 'Invalid sharelink ID: %s' % share_id}) return (output_objects, returnvalues.CLIENT_ERROR) # TODO: load and check sharelink pickle (currently requires client_id) if share_mode == 'write-only': logger.error('%s called import from write-only sharelink: %s' % (op_name, accepted)) output_objects.append( {'object_type': 'error_text', 'text': 'Sharelink %s is write-only!' % share_id}) return (output_objects, returnvalues.CLIENT_ERROR) target_dir = os.path.join(share_mode, share_id) src_base = os.path.abspath(os.path.join(configuration.sharelink_home, target_dir)) + os.sep if os.path.isfile(os.path.realpath(src_base)): logger.error('%s called import on single file sharelink: %s' % (op_name, share_id)) output_objects.append( {'object_type': 'error_text', 'text': """Import is only supported for directory sharelinks!"""}) return (output_objects, returnvalues.CLIENT_ERROR) elif not os.path.isdir(src_base): logger.error('%s called import with non-existant sharelink: %s' % (client_id, share_id)) output_objects.append( {'object_type': 'error_text', 'text': 'No such sharelink: %s' % share_id}) return (output_objects, returnvalues.CLIENT_ERROR) # Archive import if freeze_id is given - change to archive as src base if freeze_id: if not is_frozen_archive(client_id, freeze_id, configuration): logger.error('%s called with invalid freeze_id: %s' % (op_name, freeze_id)) output_objects.append( {'object_type': 'error_text', 'text': 'Invalid archive ID: %s' % freeze_id}) return (output_objects, returnvalues.CLIENT_ERROR) target_dir = os.path.join(client_dir, freeze_id) src_base = os.path.abspath(os.path.join(configuration.freeze_home, target_dir)) + os.sep if not os.path.isdir(src_base): logger.error('%s called import with non-existant archive: %s' % (client_id, freeze_id)) output_objects.append( {'object_type': 'error_text', 'text': 'No such archive: %s' % freeze_id}) return (output_objects, returnvalues.CLIENT_ERROR) status = returnvalues.OK abs_dest = dst_base + dst dst_list = glob.glob(abs_dest) if not dst_list: # New destination? if not glob.glob(os.path.dirname(abs_dest)): logger.error('%s called with illegal dst: %s' % (op_name, dst)) output_objects.append( {'object_type': 'error_text', 'text': 'Illegal dst path provided!'}) return (output_objects, returnvalues.CLIENT_ERROR) else: dst_list = [abs_dest] # Use last match in case of multiple matches dest = dst_list[-1] if len(dst_list) > 1: output_objects.append( {'object_type': 'warning', 'text': 'dst (%s) matches multiple targets - using last: %s' % (dst, dest)}) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(dest) # Don't use abs_path in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(dst_base, '') if not valid_user_path(configuration, abs_dest, dst_base, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dest, dst)) output_objects.append( {'object_type': 'error_text', 'text': "Invalid destination (%s expands to an illegal path)" % dst}) return (output_objects, returnvalues.CLIENT_ERROR) # We must make sure target dir exists if called in import X mode if (share_id or freeze_id) and not makedirs_rec(abs_dest, configuration): logger.error('could not create import destination dir: %s' % abs_dest) output_objects.append( {'object_type': 'error_text', 'text': 'cannot import to "%s" : file in the way?' % relative_dest}) return (output_objects, returnvalues.SYSTEM_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_dest)) output_objects.append( {'object_type': 'error_text', 'text': 'cannot copy to "%s": inside a read-only location!' % relative_dest}) return (output_objects, returnvalues.CLIENT_ERROR) if share_id and not force(flags) and not check_empty_dir(abs_dest): logger.warning('%s called %s sharelink import with non-empty dst: %s' % (op_name, share_id, abs_dest)) output_objects.append( {'object_type': 'error_text', 'text': """Importing a sharelink like '%s' into the non-empty '%s' folder will potentially overwrite existing files with the sharelink version. If you really want that, please try import again and select the overwrite box to confirm it. You may want to back up any important data from %s first, however. """ % (share_id, relative_dest, relative_dest)}) return (output_objects, returnvalues.CLIENT_ERROR) if freeze_id and not force(flags) and not check_empty_dir(abs_dest): logger.warning('%s called %s archive import with non-empty dst: %s' % (op_name, freeze_id, abs_dest)) output_objects.append( {'object_type': 'error_text', 'text': """Importing an archive like '%s' into the non-empty '%s' folder will potentially overwrite existing files with the archive version. If you really want that, please try import again and select the overwrite box to confirm it. You may want to back up any important data from %s first, however. """ % (freeze_id, relative_dest, relative_dest)}) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in src_list: unfiltered_match = glob.glob(src_base + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, src_base, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(src_base, '') if verbose(flags): output_objects.append( {'object_type': 'file', 'name': relative_path}) # Prevent vgrid share copy which would create read-only dot dirs # Generally refuse handling symlinks including root vgrid shares if os.path.islink(abs_path): output_objects.append( {'object_type': 'warning', 'text': """You're not allowed to copy entire special folders like %s shared folders!""" % configuration.site_vgrid_label}) status = returnvalues.CLIENT_ERROR continue elif os.path.realpath(abs_path) == os.path.realpath(base_dir): logger.error("%s: refusing copy home dir: %s" % (op_name, abs_path)) output_objects.append( {'object_type': 'warning', 'text': "You're not allowed to copy your entire home directory!" }) status = returnvalues.CLIENT_ERROR continue # src must be a file unless recursive is specified if not recursive(flags) and os.path.isdir(abs_path): logger.warning('skipping directory source %s' % abs_path) output_objects.append( {'object_type': 'warning', 'text': 'skipping directory src %s!' % relative_path}) continue # If destination is a directory the src should be copied there abs_target = abs_dest if os.path.isdir(abs_target): abs_target = os.path.join(abs_target, os.path.basename(abs_path)) if os.path.abspath(abs_path) == os.path.abspath(abs_target): logger.warning('%s tried to %s %s to itself! (%s)' % (client_id, op_name, abs_path, pattern)) output_objects.append( {'object_type': 'warning', 'text': "Cannot copy '%s' to self!" % relative_path}) status = returnvalues.CLIENT_ERROR continue if os.path.isdir(abs_path) and \ abs_target.startswith(abs_path + os.sep): logger.warning('%s tried to %s %s to itself! (%s)' % (client_id, op_name, abs_path, pattern)) output_objects.append( {'object_type': 'warning', 'text': "Cannot copy '%s' to (sub) self!" % relative_path}) status = returnvalues.CLIENT_ERROR continue try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'copied', [relative_path, relative_dest + "/" + os.path.basename(relative_path)]) if os.path.isdir(abs_path): shutil.copytree(abs_path, abs_target) else: shutil.copy(abs_path, abs_target) logger.info('%s %s %s done' % (op_name, abs_path, abs_target)) except Exception as exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'copied', [relative_path, relative_dest + "/" + os.path.basename(relative_path)], failed=True, details=exc) output_objects.append( {'object_type': 'error_text', 'text': "%s: failed on '%s' to '%s'" % (op_name, relative_path, relative_dest)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR return (output_objects, status)
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: src and dst can use wildcards here typecheck_overrides={ 'src': valid_path_pattern, 'dst': valid_path_pattern }, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) src_list = accepted['src'] dst = accepted['dst'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK abs_dest = base_dir + dst dst_list = glob.glob(abs_dest) if not dst_list: # New destination? if not glob.glob(os.path.dirname(abs_dest)): output_objects.append({ 'object_type': 'error_text', 'text': 'Illegal dst path provided!' }) return (output_objects, returnvalues.CLIENT_ERROR) else: dst_list = [abs_dest] # Use last match in case of multiple matches dest = dst_list[-1] if len(dst_list) > 1: output_objects.append({ 'object_type': 'warning', 'text': 'dst (%s) matches multiple targets - using last: %s' % (dst, dest) }) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(dest) # Don't use abs_path in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): logger.warning('%s tried to %s to restricted path %s ! (%s)' % (client_id, op_name, abs_dest, dst)) output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % dst }) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_dest)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot move to "%s": inside a read-only location!' % relative_dest }) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in src_list: unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: no such file or directory! %s' % (op_name, pattern) }) status = returnvalues.CLIENT_ERROR for abs_path in match: relative_path = abs_path.replace(base_dir, '') if verbose(flags): output_objects.append({ 'object_type': 'file', 'name': relative_path }) # Generally refuse handling symlinks including root vgrid shares if os.path.islink(abs_path): output_objects.append({ 'object_type': 'warning', 'text': """You're not allowed to move entire special folders like %s shared folders!""" % configuration.site_vgrid_label }) status = returnvalues.CLIENT_ERROR continue # Additionally refuse operations on inherited subvgrid share roots elif in_vgrid_share(configuration, abs_path) == relative_path: output_objects.append({ 'object_type': 'warning', 'text': """You're not allowed to move entire %s shared folders!""" % configuration.site_vgrid_label }) status = returnvalues.CLIENT_ERROR continue elif os.path.realpath(abs_path) == os.path.realpath(base_dir): logger.error("%s: refusing move home dir: %s" % (op_name, abs_path)) output_objects.append({ 'object_type': 'warning', 'text': "You're not allowed to move your entire home directory!" }) status = returnvalues.CLIENT_ERROR continue if not check_write_access(abs_path): logger.warning('%s called without write access: %s' % (op_name, abs_path)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot move "%s": inside a read-only location!' % pattern }) status = returnvalues.CLIENT_ERROR continue # If destination is a directory the src should be moved in there # Move with existing directory as target replaces the directory! abs_target = abs_dest if os.path.isdir(abs_target): if os.path.samefile(abs_target, abs_path): output_objects.append({ 'object_type': 'warning', 'text': "Cannot move '%s' to a subdirectory of itself!" % relative_path }) status = returnvalues.CLIENT_ERROR continue abs_target = os.path.join(abs_target, os.path.basename(abs_path)) try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'moved', [relative_path, relative_dest]) shutil.move(abs_path, abs_target) logger.info('%s %s %s done' % (op_name, abs_path, abs_target)) except Exception as exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'moved', [relative_path, relative_dest], failed=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False, op_menu=client_id) defaults = signature()[1] (validate_status, accepted) = validate_input( user_arguments_dict, defaults, output_objects, allow_rejects=False, # NOTE: path can use wildcards, current_dir cannot typecheck_overrides={'path': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) pattern_list = accepted['path'] current_dir = accepted['current_dir'][-1].lstrip('/') share_id = accepted['share_id'][-1] show_dest = accepted['with_dest'][0].lower() == 'true' status = returnvalues.OK # NOTE: in contrast to 'ls' we never include write operations here read_mode, write_mode = True, False visibility_mods = ''' .%(main_class)s .enable_write { display: none; } .%(main_class)s .disable_read { display: none; } .%(main_class)s .if_full { display: none; } ''' # Either authenticated user client_id set or sharelink ID if client_id: user_id = client_id target_dir = client_id_dir(client_id) base_dir = configuration.user_home redirect_name = configuration.site_user_redirect redirect_path = redirect_name id_args = '' root_link_name = 'USER HOME' main_class = "user_expand" page_title = 'User Files - Path Expansion' userstyle = True widgets = True elif share_id: try: (share_mode, _) = extract_mode_id(configuration, share_id) except ValueError as err: logger.error('%s called with invalid share_id %s: %s' % (op_name, share_id, err)) output_objects.append( {'object_type': 'error_text', 'text': 'Invalid sharelink ID: %s' % share_id}) return (output_objects, returnvalues.CLIENT_ERROR) # TODO: load and check sharelink pickle (currently requires client_id) # then include shared by %(owner)s on page header user_id = 'anonymous user through share ID %s' % share_id target_dir = os.path.join(share_mode, share_id) base_dir = configuration.sharelink_home redirect_name = 'share_redirect' redirect_path = os.path.join(redirect_name, share_id) id_args = 'share_id=%s;' % share_id root_link_name = '%s' % share_id main_class = "sharelink_expand" page_title = 'Shared Files - Path Expansion' userstyle = False widgets = False else: logger.error('%s called without proper auth: %s' % (op_name, accepted)) output_objects.append({'object_type': 'error_text', 'text': 'Authentication is missing!' }) return (output_objects, returnvalues.SYSTEM_ERROR) visibility_toggle = ''' <style> %s </style> ''' % (visibility_mods % {'main_class': main_class}) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(base_dir, target_dir)) + os.sep if not os.path.isdir(base_dir): logger.error('%s called on missing base_dir: %s' % (op_name, base_dir)) output_objects.append({'object_type': 'error_text', 'text': 'No such %s!' % page_title.lower() }) return (output_objects, returnvalues.CLIENT_ERROR) title_entry = find_entry(output_objects, 'title') title_entry['text'] = page_title title_entry['skipwidgets'] = not widgets title_entry['skipuserstyle'] = not userstyle fill_helpers = {'dest_dir': current_dir + os.sep, 'share_id': share_id, 'flags': flags, 'tmp_flags': flags, 'long_set': long_list(flags), 'recursive_set': recursive(flags), 'all_set': all(flags)} add_import, add_init, add_ready = '', '', '' title_entry['style']['advanced'] += ''' %s ''' % visibility_toggle title_entry['script']['advanced'] += add_import title_entry['script']['init'] += add_init title_entry['script']['ready'] += add_ready title_entry['script']['body'] = ' class="%s"' % main_class output_objects.append({'object_type': 'header', 'text': page_title}) # Shared URL helpers ls_url_template = 'ls.py?%scurrent_dir=%%(rel_dir_enc)s;flags=%s' % \ (id_args, flags) redirect_url_template = '/%s/%%(rel_path_enc)s' % redirect_path location_pre_html = """ <div class='files'> <table class='files'> <tr class=title><td class=centertext> Working directory: </td></tr> <tr><td class='centertext'> """ output_objects.append( {'object_type': 'html_form', 'text': location_pre_html}) # Use current_dir nav location links for pattern in pattern_list[:1]: links = [] links.append({'object_type': 'link', 'text': root_link_name, 'destination': ls_url_template % {'rel_dir_enc': '.'}}) prefix = '' parts = os.path.normpath(current_dir).split(os.sep) for i in parts: if i == ".": continue prefix = os.path.join(prefix, i) links.append({'object_type': 'link', 'text': i, 'destination': ls_url_template % {'rel_dir_enc': quote(prefix)}}) output_objects.append( {'object_type': 'multilinkline', 'links': links, 'sep': ' %s ' % os.sep}) location_post_html = """ </td></tr> </table> </div> <br /> """ output_objects.append( {'object_type': 'html_form', 'text': location_post_html}) dir_listings = [] output_objects.append({ 'object_type': 'dir_listings', 'dir_listings': dir_listings, 'flags': flags, 'redirect_name': redirect_name, 'redirect_path': redirect_path, 'share_id': share_id, 'ls_url_template': ls_url_template, 'rm_url_template': '', 'rmdir_url_template': '', 'editor_url_template': '', 'redirect_url_template': redirect_url_template, 'show_dest': show_dest, }) first_match = None for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages current_path = os.path.normpath(os.path.join(base_dir, current_dir)) unfiltered_match = glob.glob(current_path + os.sep + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (user_id, op_name, abs_path, pattern)) continue match.append(abs_path) if not first_match: first_match = abs_path # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name': pattern}) status = returnvalues.FILE_NOT_FOUND for abs_path in match: if abs_path + os.sep == base_dir: relative_path = '.' else: relative_path = abs_path.replace(base_dir, '') entries = [] dir_listing = { 'object_type': 'dir_listing', 'relative_path': relative_path, 'entries': entries, 'flags': flags, } dest = '' if show_dest: if os.path.isfile(abs_path): dest = os.path.basename(abs_path) elif recursive(flags): # references to '.' or similar are stripped by abspath if abs_path + os.sep == base_dir: dest = '' else: # dest = os.path.dirname(abs_path).replace(base_dir, "") dest = os.path.basename(abs_path) + os.sep handle_expand(configuration, output_objects, entries, base_dir, abs_path, flags, dest, 0, show_dest) dir_listings.append(dir_listing) output_objects.append({'object_type': 'html_form', 'text': """ <div class='files disable_read'> <form method='get' action='ls.py'> <table class='files'> <tr class=title><td class=centertext> Filter paths (wildcards like * and ? are allowed) <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> <input type='text' name='path' value='' /> <input type='submit' value='Filter' /> </td></tr> </table> </form> </div> """ % fill_helpers}) # Short/long format buttons fill_helpers['tmp_flags'] = flags + 'l' htmlform = """ <table class='files if_full'> <tr class=title><td class=centertext colspan=4> File view options </td></tr> <tr><td colspan=4><br /></td></tr> <tr class=title><td>Parameter</td><td>Setting</td><td>Enable</td><td>Disable</td></tr> <tr><td>Long format</td><td> %(long_set)s</td><td> <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />" % entry fill_helpers['tmp_flags'] = flags.replace('l', '') htmlform += """ <input type='submit' value='On' /><br /> </form> </td><td> <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />" % entry htmlform += """ <input type='submit' value='Off' /><br /> </form> </td></tr> """ # Recursive output fill_helpers['tmp_flags'] = flags + 'r' htmlform += """ <!-- Non-/recursive list buttons --> <tr><td>Recursion</td><td> %(recursive_set)s</td><td>""" % fill_helpers htmlform += """ <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />" % entry fill_helpers['tmp_flags'] = flags.replace('r', '') htmlform += """ <input type='submit' value='On' /><br /> </form> </td><td> <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />"\ % entry htmlform += """ <input type='submit' value='Off' /><br /> </form> </td></tr> """ htmlform += """ <!-- Show dot files buttons --> <tr><td>Show hidden files</td><td> %(all_set)s</td><td>""" % fill_helpers fill_helpers['tmp_flags'] = flags + 'a' htmlform += """ <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />" % entry fill_helpers['tmp_flags'] = flags.replace('a', '') htmlform += """ <input type='submit' value='On' /><br /> </form> </td><td> <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />" % entry htmlform += """ <input type='submit' value='Off' /><br /> </form> </td></tr> </table> """ # show flag buttons after contents to limit clutter output_objects.append({'object_type': 'html_form', 'text': htmlform}) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: path can use wildcards typecheck_overrides={'path': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] name_pattern = accepted['name'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) # Shared URL helpers id_args = '' redirect_name = configuration.site_user_redirect redirect_path = redirect_name form_method = 'post' csrf_limit = get_csrf_limit(configuration) ls_url_template = 'ls.py?%scurrent_dir=%%(rel_dir_enc)s;flags=%s' % \ (id_args, flags) csrf_token = make_csrf_token(configuration, form_method, 'rm', client_id, csrf_limit) rm_url_template = 'rm.py?%spath=%%(rel_path_enc)s;%s=%s' % \ (id_args, csrf_field, csrf_token) rmdir_url_template = 'rm.py?%spath=%%(rel_path_enc)s;flags=r;%s=%s' % \ (id_args, csrf_field, csrf_token) editor_url_template = 'editor.py?%spath=%%(rel_path_enc)s' % id_args redirect_url_template = '/%s/%%(rel_path_enc)s' % redirect_path dir_listings = [] output_objects.append({ 'object_type': 'dir_listings', 'dir_listings': dir_listings, 'flags': flags, 'ls_url_template': ls_url_template, 'rm_url_template': rm_url_template, 'rmdir_url_template': rmdir_url_template, 'editor_url_template': editor_url_template, 'redirect_url_template': redirect_url_template, }) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') entries = [] dir_listing = { 'object_type': 'dir_listing', 'relative_path': relative_path, 'entries': entries, 'flags': flags, } dir_listings.append(dir_listing) try: for (root, dirs, files) in walk(abs_path): for filename in fnmatch.filter(files, name_pattern): # IMPORTANT: this join always yields abs expanded path abs_path = os.path.join(root, filename) relative_path = abs_path.replace(base_dir, '') if not valid_user_path(configuration, abs_path, base_dir, True): continue file_with_dir = relative_path file_obj = { 'object_type': 'direntry', 'type': 'file', 'name': filename, 'rel_path': file_with_dir, 'rel_path_enc': quote(file_with_dir), 'rel_dir_enc': quote(os.path.dirname(file_with_dir)), # NOTE: file_with_dir is kept for backwards compliance 'file_with_dir': file_with_dir, 'flags': flags, 'special': '', } entries.append(file_obj) except Exception as exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if verbose(flags): output_objects.append({ 'object_type': 'file_output', 'path': relative_path, 'lines': output_lines }) else: output_objects.append({ 'object_type': 'file_output', 'lines': output_lines }) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] status = returnvalues.OK (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: src can use wildcards, dst cannot typecheck_overrides={'src': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) job_ids = accepted['job_id'] action = accepted['action'][-1] src = accepted['src'] dst = accepted['dst'][-1] title_entry = find_entry(output_objects, 'title') title_entry['text'] = '%s live I/O' % configuration.short_title add_import, add_init, add_ready = '', '', '' add_init += ''' var fields = 1; var max_fields = 20; var src_input = "<input class='fillwidth' type=text name=src value='' /><br />"; function addSource() { if (fields < max_fields) { $("#srcfields").append(src_input); fields += 1; } else { alert("Maximum " + max_fields + " source fields allowed!"); } } ''' title_entry['script']['advanced'] += add_import title_entry['script']['init'] += add_init title_entry['script']['ready'] += add_ready output_objects.append({ 'object_type': 'header', 'text': 'Request live communication with jobs' }) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) if not action in valid_actions: output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid action "%s" (supported: %s)' % (action, ', '.join(valid_actions)) }) return (output_objects, returnvalues.CLIENT_ERROR) if action in post_actions: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not job_ids or action in interactive_actions: job_id = '' if job_ids: job_id = job_ids[-1] form_method = 'post' csrf_limit = get_csrf_limit(configuration) fill_helpers = { 'job_id': job_id, 'form_method': form_method, 'csrf_field': csrf_field, 'csrf_limit': csrf_limit } target_op = 'liveio' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token}) output_objects.append({ 'object_type': 'text', 'text': ''' Fill in the live I/O details below to request communication with a running job. Job ID can be a full ID or a wild card pattern using "*" and "?" to match one or more of your job IDs. Use send output without source and destination paths to request upload of the default stdio files from the job on the resource to the associated job_output directory in your MiG home. Destination is a always handled as a directory path to put source files into. Source and destination paths are always taken relative to the job execution directory on the resource and your MiG home respectively. ''' }) html = ''' <table class="liveio"> <tr> <td> <form method="%(form_method)s" action="%(target_op)s.py"> <table class="liveio"> <tr><td class=centertext> </td></tr> <tr><td> Action:<br /> <input type="hidden" name="%(csrf_field)s" value="%(csrf_token)s" /> <input type=radio name=action checked value="send" />send output <input type=radio name=action value="get" />get input </td></tr> <tr><td> Job ID:<br /> <input class="fillwidth" type=text name=job_id value="%(job_id)s" /> </td></tr> <tr><td> Source path(s):<br /> <div id="srcfields"> <input class="fillwidth" type=text name=src value="" /><br /> </div> <input id="addsrcbutton" type="button" onclick="addSource(); return false;" value="Add another source field" /> </td></tr> <tr><td> Destination path:<br /> <input class="fillwidth" type=text name=dst value="" /> </td></tr> <tr><td> <input type="submit" value="Send request" /> </td></tr> </table> </form> </td> </tr> </table> ''' % fill_helpers output_objects.append({'object_type': 'html_form', 'text': html}) output_objects.append({ 'object_type': 'text', 'text': ''' Further live job control is avalable through your personal message queues. They provide a basic interface for centrally storing messages under your grid account and can be used to pass messages between jobs or for orchestrating jobs before and during execution. ''' }) output_objects.append({ 'object_type': 'link', 'destination': 'mqueue.py', 'text': 'Message queue interface' }) return (output_objects, returnvalues.OK) elif action in ['get', 'receive', 'input']: action = 'get' action_desc = 'will be downloaded to the job on the resource' elif action in ['put', 'send', 'output']: action = 'send' action_desc = 'will be uploaded from the job on the resource' else: output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid live io action: %s' % action }) return (output_objects, returnvalues.CLIENT_ERROR) output_objects.append({ 'object_type': 'text', 'text': 'Requesting live I/O for %s' % ', '.join(job_ids) }) if action == 'get' and (not src or not dst): output_objects.append({ 'object_type': 'error_text', 'text': 'src and dst parameters required for live input' }) return (output_objects, returnvalues.CLIENT_ERROR) # Automatic fall back to stdio files if output with no path provided if src: src_text = 'The files ' + ' '.join(src) else: src_text = 'The job stdio files' if dst: dst_text = 'the ' + dst + ' directory' else: dst_text = 'the corresponding job_output directory' # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep filelist = [] for job_id in job_ids: job_id = job_id.strip() # is job currently being executed? # Backward compatibility - all_jobs keyword should match all jobs if job_id == all_jobs: job_id = '*' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + job_id + '.mRSL') match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning("%s tried to %s restricted path %s ! (%s)" % (client_id, op_name, abs_path, job_id)) continue # Insert valid job files in filelist for later treatment match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match.... if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % job_id }) else: filelist += match for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') job_dict = unpickle(filepath, logger) if not job_dict: status = returnvalues.CLIENT_ERROR output_objects.append({ 'object_type': 'error_text', 'text': ('You can only list status of your own jobs. ' 'Please verify that you submitted the mRSL file ' 'with job id "%s" (Could not unpickle mRSL file %s)') % (job_id, filepath) }) continue if job_dict['STATUS'] != 'EXECUTING': output_objects.append({ 'object_type': 'text', 'text': 'Job %s is not currently being executed! Job status: %s' % (job_id, job_dict['STATUS']) }) continue if job_dict['UNIQUE_RESOURCE_NAME'] == 'ARC': output_objects.append({ 'object_type': 'text', 'text': 'Job %s is submitted to ARC, details are not available!' % job_id }) continue last_live_update_dict = {} last_live_update_file = configuration.mig_system_files + os.sep\ + job_id + '.last_live_update' if os.path.isfile(last_live_update_file): last_live_update_dict_unpickled = \ unpickle(last_live_update_file, logger) if not last_live_update_dict_unpickled: output_objects.append({ 'object_type': 'error_text', 'text': 'Could not unpickle %s - skipping request!' % last_live_update_file }) continue if 'LAST_LIVE_UPDATE_REQUEST_TIMESTAMP' not in last_live_update_dict_unpickled: output_objects.append({ 'object_type': 'error_text', 'text': 'Could not find needed key in %s.' % last_live_update_file }) continue last_live_update_request = \ last_live_update_dict_unpickled['LAST_LIVE_UPDATE_REQUEST_TIMESTAMP' ] difference = datetime.datetime.now() - last_live_update_request try: min_delay = \ int(configuration.min_seconds_between_live_update_requests) except: min_delay = 30 if difference.seconds < min_delay: output_objects.append({ 'object_type': 'error_text', 'text': ('Request not allowed, you must wait at least ' '%s seconds between live update requests!') % min_delay }) continue # save this request to file to avoid DoS from a client request loop. last_live_update_dict['LAST_LIVE_UPDATE_REQUEST_TIMESTAMP'] = \ datetime.datetime.now() pickle_ret = pickle(last_live_update_dict, last_live_update_file, logger) if not pickle_ret: output_objects.append({ 'object_type': 'error_text', 'text': 'Error saving live io request timestamp to last_live_update ' 'file, request not sent!' }) continue # # # ## job is being executed right now, send live io request to frontend # # # get resource_config, needed by scp_file_to_resource # (res_status, resource_config) = get_resource_configuration( # resource_home, unique_resource_name, logger) resource_config = job_dict['RESOURCE_CONFIG'] (res_status, exe) = get_resource_exe(resource_config, job_dict['EXE'], logger) if not res_status: output_objects.append({ 'object_type': 'error_text', 'text': 'Could not get exe configuration for job %s' % job_id }) continue local_file = '%s.%supdate' % (job_dict['LOCALJOBNAME'], action) if not os.path.exists(local_file): # create try: filehandle = open(local_file, 'w') filehandle.write('job_id ' + job_dict['JOB_ID'] + '\n') filehandle.write('localjobname ' + job_dict['LOCALJOBNAME'] + '\n') filehandle.write('execution_user ' + exe['execution_user'] + '\n') filehandle.write('execution_node ' + exe['execution_node'] + '\n') filehandle.write('execution_dir ' + exe['execution_dir'] + '\n') filehandle.write('target liveio\n') # Leave defaults src and dst to FE script if not provided if src: filehandle.write('source ' + ' '.join(src) + '\n') if dst: filehandle.write('destination ' + dst + '\n') # Backward compatible test for shared_fs - fall back to scp if 'shared_fs' in exe and exe['shared_fs']: filehandle.write('copy_command cp\n') filehandle.write('copy_frontend_prefix \n') filehandle.write('copy_execution_prefix \n') else: filehandle.write('copy_command scp -B\n') filehandle.write( 'copy_frontend_prefix ${frontend_user}@${frontend_node}:\n' ) filehandle.write( 'copy_execution_prefix ${execution_user}@${execution_node}:\n' ) filehandle.write('### END OF SCRIPT ###\n') filehandle.close() except Exception as exc: pass if not os.path.exists(local_file): output_objects.append({ 'object_type': 'error_text', 'text': '.%supdate file not available on %s server' % (action, configuration.short_title) }) continue scp_status = copy_file_to_resource( local_file, '%s.%supdate' % (job_dict['LOCALJOBNAME'], action), resource_config, logger) if not scp_status: output_objects.append({ 'object_type': 'error_text', 'text': 'Error sending request for live io to resource!' }) continue else: output_objects.append({ 'object_type': 'text', 'text': 'Request for live io was successfully sent to the resource!' }) output_objects.append({ 'object_type': 'text', 'text': '%s %s and should become available in %s in a minute.' % (src_text, action_desc, dst_text) }) if action == 'send': if not dst: target_path = '%s/%s/*' % (job_output_dir, job_id) else: target_path = dst enc_url = 'ls.py?path=%s' % quote(target_path) output_objects.append({ 'object_type': 'link', 'destination': enc_url, 'text': 'View uploaded files' }) else: enc_url = 'ls.py?path=' enc_url += ';path='.join([quote(i) for i in src]) output_objects.append({ 'object_type': 'link', 'destination': enc_url, 'text': 'View files for download' }) try: os.remove(local_file) except Exception as exc: pass return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) valid_langs = {'sh': 'shell', 'python': 'python'} valid_flavors = {'user': '******', 'resource': 'vgridscriptgen'} defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) langs = accepted['lang'] flavor_list = accepted['flavor'] sh_cmd = accepted['sh_cmd'][-1] python_cmd = accepted['python_cmd'][-1] script_dir = accepted['script_dir'][-1] flavors = [] title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'Script generator' output_objects.append( {'object_type': 'header', 'text': 'Script generator'}) status = returnvalues.OK # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep if 'h' in flags: output_objects = usage(output_objects, valid_langs, valid_flavors) return (output_objects, status) if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append( {'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Filter out any invalid flavors to avoid illegal filenames, etc. for f in flavor_list: if f in valid_flavors.keys(): flavors.append(f) # Default to user scripts if not flavors: if flavor_list: output_objects.append({'object_type': 'text', 'text': 'No valid flavors specified - falling back to user scripts' }) flavors = ['user'] if not langs or keyword_all in langs: # Add new languages here languages = [(userscriptgen.sh_lang, sh_cmd, userscriptgen.sh_ext), (userscriptgen.python_lang, python_cmd, userscriptgen.python_ext)] else: languages = [] # check arguments for lang in langs: if lang == 'sh': interpreter = sh_cmd extension = userscriptgen.sh_ext elif lang == 'python': interpreter = python_cmd extension = userscriptgen.python_ext else: output_objects.append({'object_type': 'warning', 'text': 'Unknown script language: %s - ignoring!' % lang}) continue languages.append((lang, interpreter, extension)) if not languages: output_objects.append({'object_type': 'error_text', 'text': 'No valid languages specified - aborting script generation' }) return (output_objects, returnvalues.CLIENT_ERROR) for flavor in flavors: if not script_dir or script_dir == keyword_auto: # Generate scripts in a "unique" destination directory # gmtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min, # tm_sec, tm_wday, tm_yday, tm_isdst) now = time.gmtime() timestamp = '%.2d%.2d%.2d-%.2d%.2d%.2d' % ( now[2], now[1], now[0], now[3], now[4], now[5], ) script_dir = '%s-%s-scripts-%s' % (configuration.short_title, flavor, timestamp) else: # Avoid problems from especially trailing slash (zip recursion) script_dir = script_dir.strip(os.sep) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dir = os.path.abspath(os.path.join(base_dir, script_dir)) if not valid_user_path(configuration, abs_dir, base_dir, True): # out of bounds output_objects.append({'object_type': 'error_text', 'text': "You're not allowed to work in %s!" % script_dir}) logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dir, script_dir)) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(abs_dir): try: os.mkdir(abs_dir) except Exception as exc: output_objects.append({'object_type': 'error_text', 'text': 'Failed to create destination directory (%s) - aborting script generation' % exc}) return (output_objects, returnvalues.SYSTEM_ERROR) for (lang, _, _) in languages: output_objects.append({'object_type': 'text', 'text': 'Generating %s %s scripts in the %s subdirectory of your %s home directory' % (lang, flavor, script_dir, configuration.short_title)}) logger.debug('generate %s scripts in %s' % (flavor, abs_dir)) # Generate all scripts if flavor == 'user': for op in userscriptgen.script_ops: generator = 'userscriptgen.generate_%s' % op eval(generator)(configuration, languages, abs_dir) if userscriptgen.shared_lib: userscriptgen.generate_lib(configuration, userscriptgen.script_ops, languages, abs_dir) if userscriptgen.test_script: userscriptgen.generate_test(configuration, languages, abs_dir) elif flavor == 'resource': for op in vgridscriptgen.script_ops_single_arg: vgridscriptgen.generate_single_argument(configuration, op[0], op[1], languages, abs_dir) for op in vgridscriptgen.script_ops_single_upload_arg: vgridscriptgen.generate_single_argument_upload(configuration, op[0], op[1], op[2], languages, abs_dir) for op in vgridscriptgen.script_ops_two_args: vgridscriptgen.generate_two_arguments(configuration, op[0], op[1], op[2], languages, abs_dir) for op in vgridscriptgen.script_ops_ten_args: vgridscriptgen.generate_ten_arguments(configuration, op[0], op[1], op[2], op[3], op[4], op[5], op[6], op[7], op[8], op[9], op[10], languages, abs_dir) else: output_objects.append( {'object_type': 'warning_text', 'text': 'Unknown flavor: %s' % flavor}) continue # Always include license conditions file userscriptgen.write_license(configuration, abs_dir) output_objects.append({'object_type': 'text', 'text': '... Done' }) output_objects.append({'object_type': 'text', 'text': '%s %s scripts are now available in your %s home directory:' % (configuration.short_title, flavor, configuration.short_title)}) output_objects.append({'object_type': 'link', 'text': 'View directory', 'destination': 'fileman.py?path=%s/' % script_dir}) # Create zip from generated dir output_objects.append({'object_type': 'text', 'text': 'Generating zip archive of the %s %s scripts' % (configuration.short_title, flavor)}) script_zip = script_dir + '.zip' dest_zip = '%s%s' % (base_dir, script_zip) logger.debug('packing generated scripts from %s in %s' % (abs_dir, dest_zip)) # Force compression zip_file = zipfile.ZipFile(dest_zip, 'w', zipfile.ZIP_DEFLATED) # Directory write is not supported - add each file manually for script in os.listdir(abs_dir): zip_file.write(abs_dir + os.sep + script, script_dir + os.sep + script) # Preserve executable flag in accordance with: # http://mail.python.org/pipermail/pythonmac-sig/2005-March/013491.html for zinfo in zip_file.filelist: zinfo.create_system = 3 zip_file.close() # Verify CRC zip_file = zipfile.ZipFile(dest_zip, 'r') err = zip_file.testzip() zip_file.close() if err: output_objects.append({'object_type': 'error_text', 'text': 'Zip file integrity check failed! (%s)' % err}) status = returnvalues.SYSTEM_ERROR continue output_objects.append({'object_type': 'text', 'text': '... Done' }) output_objects.append({'object_type': 'text', 'text': 'Zip archive of the %s %s scripts are now available in your %s home directory' % (configuration.short_title, flavor, configuration.short_title)}) output_objects.append({'object_type': 'link', 'text': 'Download zip archive %s' % script_zip, 'destination': os.path.join('..', client_dir, script_zip)}) output_objects.append({'object_type': 'upgrade_info', 'text': ''' You can upgrade from an existing user scripts folder with the commands:''', 'commands': ["./migget.sh '%s' ../" % script_zip, "cd ..", "unzip '%s'" % script_zip, "cd '%s'" % script_dir] }) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False, op_menu=client_id) defaults = signature()[1] (validate_status, accepted) = validate_input(user_arguments_dict, defaults, output_objects, allow_rejects=False) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) queue = accepted['queue'][-1] action = accepted['action'][-1] iosessionid = accepted['iosessionid'][-1] msg = accepted['msg'][-1] msg_id = accepted['msg_id'][-1] # Web format for cert access and no header for SID access if client_id: output_objects.append({ 'object_type': 'header', 'text': 'Message queue %s' % action }) else: output_objects.append({'object_type': 'start'}) # Always return at least a basic file_output entry file_entry = { 'object_type': 'file_output', 'lines': [], 'wrap_binary': True, 'wrap_targets': ['lines'] } if not action in valid_actions: output_objects.append({'object_type': 'error_text', 'text' : 'Invalid action "%s" (supported: %s)' % \ (action, ', '.join(valid_actions))}) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) if action in post_actions: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Find user home from session or certificate if iosessionid: client_home = os.path.realpath( os.path.join(configuration.webserver_home, iosessionid)) client_dir = os.path.basename(client_home) elif client_id: client_dir = client_id_dir(client_id) else: output_objects.append({ 'object_type': 'error_text', 'text': 'Either certificate or session ID is required' }) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if not os.path.isdir(base_dir): output_objects.append({ 'object_type': 'error_text', 'text': 'No matching session or user home!' }) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) mqueue_base = os.path.join(base_dir, mqueue_prefix) + os.sep default_queue_dir = os.path.join(mqueue_base, default_mqueue) # Create mqueue base and default queue dir if missing if not os.path.exists(default_queue_dir): try: os.makedirs(default_queue_dir) except: pass # IMPORTANT: path must be expanded to abs for proper chrooting queue_path = os.path.abspath(os.path.join(mqueue_base, queue)) if not valid_user_path(configuration, queue_path, mqueue_base): output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid queue name: "%s"' % queue }) output_objects.append(file_entry) return (output_objects, returnvalues.CLIENT_ERROR) lock_path = os.path.join(mqueue_base, lock_name) lock_handle = open(lock_path, 'a') fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) status = returnvalues.OK if action == "interactive": form_method = 'post' csrf_limit = get_csrf_limit(configuration) fill_helpers = { 'queue': queue, 'msg': msg, 'form_method': form_method, 'csrf_field': csrf_field, 'csrf_limit': csrf_limit, } target_op = 'mqueue' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token}) output_objects.append({ 'object_type': 'text', 'text': ''' Fill in the fields below to control and access your personal message queues. Jobs can receive from and send to the message queues during execution, and use them as a means of job inter-communication. Expect message queue operations to take several seconds on the resources, however. That is, use it for tasks like orchestrating long running jobs, and not for low latency communication. ''' }) html = ''' <form name="mqueueform" method="%(form_method)s" action="%(target_op)s.py"> <table class="mqueue"> <tr><td class=centertext> </td></tr> <tr><td> Action:<br /> <input type="hidden" name="%(csrf_field)s" value="%(csrf_token)s" /> <input type=radio name=action value="create" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />create queue <input type=radio name=action checked value="send" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=false;" />send message to queue <input type=radio name=action value="receive" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />receive message from queue <input type=radio name=action value="remove" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />remove queue <input type=radio name=action value="listqueues" onclick="javascript: document.mqueueform.queue.disabled=true; document.mqueueform.msg.disabled=true;" />list queues <input type=radio name=action value="listmessages" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />list messages <input type=radio name=action value="show" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />show message </td></tr> <tr><td> Queue:<br /> <input class="fillwidth" type=text name=queue value="%(queue)s" /> </td></tr> <tr><td> <div id="msgfieldf"> <input class="fillwidth" type=text name=msg value="%(msg)s" /><br /> </div> </td></tr> <tr><td> <input type="submit" value="Apply" /> </td></tr> </table> </form> ''' % fill_helpers output_objects.append({'object_type': 'html_form', 'text': html}) output_objects.append({ 'object_type': 'text', 'text': ''' Further live job control is avalable through the live I/O interface. They provide a basic interface for centrally managing input and output files for active jobs. ''' }) output_objects.append({ 'object_type': 'link', 'destination': 'liveio.py', 'text': 'Live I/O interface' }) return (output_objects, returnvalues.OK) elif action == 'create': try: os.mkdir(queue_path) output_objects.append({ 'object_type': 'text', 'text': 'New "%s" queue created' % queue }) except Exception as err: output_objects.append({'object_type': 'error_text', 'text' : 'Could not create "%s" queue: "%s"' % \ (queue, err)}) status = returnvalues.CLIENT_ERROR elif action == 'remove': try: for entry in os.listdir(queue_path): os.remove(os.path.join(queue_path, entry)) os.rmdir(queue_path) output_objects.append({ 'object_type': 'text', 'text': 'Existing "%s" queue removed' % queue }) except Exception as err: output_objects.append({'object_type': 'error_text', 'text' : 'Could not remove "%s" queue: "%s"' % \ (queue, err)}) status = returnvalues.CLIENT_ERROR elif action == 'send': try: if not msg_id: msg_id = "%.0f" % time.time() msg_path = os.path.join(queue_path, msg_id) msg_fd = open(msg_path, 'w') msg_fd.write(msg) msg_fd.close() output_objects.append({ 'object_type': 'text', 'text': 'Message sent to "%s" queue' % queue }) except Exception as err: output_objects.append({'object_type': 'error_text', 'text' : 'Could not send to "%s" queue: "%s"' % \ (queue, err)}) status = returnvalues.CLIENT_ERROR elif action == 'receive': try: if not msg_id: messages = os.listdir(queue_path) messages.sort() if messages: msg_id = messages[0] if msg_id: message_path = os.path.join(queue_path, msg_id) message_fd = open(message_path, 'r') message = message_fd.readlines() message_fd.close() os.remove(message_path) file_entry['path'] = os.path.basename(message_path) else: message = [mqueue_empty] # Update file_output entry for raw data with output_format=file file_entry['lines'] = message except Exception as err: output_objects.append({ 'object_type': 'error_text', 'text': 'Could not receive from "%s" queue: "%s"' % (queue, err) }) status = returnvalues.CLIENT_ERROR elif action == 'show': try: if not msg_id: messages = os.listdir(queue_path) messages.sort() if messages: msg_id = messages[0] if msg_id: message_path = os.path.join(queue_path, msg_id) message_fd = open(message_path, 'r') message = message_fd.readlines() message_fd.close() file_entry['path'] = os.path.basename(message_path) else: message = [mqueue_empty] # Update file_output entry for raw data with output_format=file file_entry['lines'] = message except Exception as err: output_objects.append({ 'object_type': 'error_text', 'text': 'Could not show %s from "%s" queue: "%s"' % (msg_id, queue, err) }) status = returnvalues.CLIENT_ERROR elif action == 'listmessages': try: messages = os.listdir(queue_path) messages.sort() output_objects.append({'object_type': 'list', 'list': messages}) except Exception as err: output_objects.append({'object_type': 'error_text', 'text' : 'Could not list "%s" queue: "%s"' % \ (queue, err)}) status = returnvalues.CLIENT_ERROR elif action == 'listqueues': try: queues = [i for i in os.listdir(mqueue_base) if \ os.path.isdir(os.path.join(mqueue_base, i))] queues.sort() output_objects.append({'object_type': 'list', 'list': queues}) except Exception as err: output_objects.append({ 'object_type': 'error_text', 'text': 'Could not list queues: "%s"' % err }) status = returnvalues.CLIENT_ERROR else: output_objects.append({ 'object_type': 'error_text', 'text': 'Unexpected mqueue action: "%s"' % action }) status = returnvalues.SYSTEM_ERROR lock_handle.close() output_objects.append(file_entry) output_objects.append({ 'object_type': 'link', 'destination': 'mqueue.py?queue=%s;msg=%s' % (queue, msg), 'text': 'Back to message queue interaction' }) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = accepted['flags'] patterns = accepted['job_id'] if not configuration.site_enable_jobs: output_objects.append({'object_type': 'error_text', 'text': '''Job execution is not enabled on this system'''}) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep mrsl_keywords_dict = get_keywords_dict(configuration) if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text' : '%s using flag: %s' % (op_name, flag)}) for pattern in patterns: # Add file extension pattern += '.mRSL' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({'object_type': 'file_not_found', 'name' : pattern}) status = returnvalues.FILE_NOT_FOUND for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') try: mrsl_dict = unpickle(abs_path, logger) if not mrsl_dict: raise Exception('could not load job mRSL') for (key, val) in mrsl_dict.items(): if not key in mrsl_keywords_dict.keys(): continue if not val: continue output_lines.append('::%s::\n' % key) if 'multiplestrings' == mrsl_keywords_dict[key]['Type']: for line in val: output_lines.append('%s\n' % line) elif 'multiplekeyvalues' == mrsl_keywords_dict[key]['Type']: for (left, right) in val: output_lines.append('%s=%s\n' % (left, right)) else: output_lines.append('%s\n' % val) output_lines.append('\n') except Exception as exc: output_objects.append({'object_type': 'error_text', 'text' : "%s: '%s': %s" % (op_name, relative_path, exc)}) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if verbose(flags): output_objects.append({'object_type': 'file_output', 'path' : relative_path, 'lines' : output_lines}) else: output_objects.append({'object_type': 'file_output', 'lines' : output_lines}) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: logger.error("jobstatus input validation failed: %s" % accepted) return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) max_jobs = int(accepted['max_jobs'][-1]) order = 'unsorted ' if sorted(flags): order = 'sorted ' patterns = accepted['job_id'] project_names = accepted['project_name'] if len(project_names) > 0: for project_name in project_names: project_name_job_ids = \ get_job_ids_with_specified_project_name(client_id, project_name, configuration.mrsl_files_dir, logger) patterns.extend(project_name_job_ids) if not configuration.site_enable_jobs: output_objects.append({'object_type': 'error_text', 'text': '''Job execution is not enabled on this system'''}) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep output_objects.append({'object_type': 'header', 'text': '%s %s job status' % (configuration.short_title, order)}) if not patterns: output_objects.append( {'object_type': 'error_text', 'text': 'No job_id specified!'}) return (output_objects, returnvalues.NO_SUCH_JOB_ID) if verbose(flags): for flag in flags: output_objects.append({'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag)}) if not os.path.isdir(base_dir): output_objects.append( {'object_type': 'error_text', 'text': ('You have not been created as a user on the %s server! ' 'Please contact the %s team.') % (configuration.short_title, configuration.short_title)}) return (output_objects, returnvalues.CLIENT_ERROR) filelist = [] for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to # avoid leaking information about file system layout while # allowing consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match.... if not match: output_objects.append( {'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % pattern}) status = returnvalues.CLIENT_ERROR else: filelist += match if sorted(flags): sort(filelist) if max_jobs > 0 and max_jobs < len(filelist): output_objects.append( {'object_type': 'text', 'text': 'Only showing first %d of the %d matching jobs as requested' % (max_jobs, len(filelist))}) filelist = filelist[:max_jobs] # Iterate through jobs and list details for each job_list = {'object_type': 'job_list', 'jobs': []} for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') job_dict = unpickle(filepath, logger) if not job_dict: status = returnvalues.CLIENT_ERROR output_objects.append( {'object_type': 'error_text', 'text': 'No such job: %s (could not load mRSL file %s)' % (job_id, filepath)}) continue # Expand any job variables before use job_dict = expand_variables(job_dict) job_obj = {'object_type': 'job', 'job_id': job_id} job_obj['status'] = job_dict['STATUS'] time_fields = [ 'VERIFIED', 'VERIFIED_TIMESTAMP', 'RECEIVED_TIMESTAMP', 'QUEUED_TIMESTAMP', 'SCHEDULE_TIMESTAMP', 'EXECUTING_TIMESTAMP', 'FINISHED_TIMESTAMP', 'FAILED_TIMESTAMP', 'CANCELED_TIMESTAMP', ] for name in time_fields: if name in job_dict: # time objects cannot be marshalled, asctime if timestamp try: job_obj[name.lower()] = time.asctime(job_dict[name]) except Exception as exc: # not a time object, just add job_obj[name.lower()] = job_dict[name] ########################################### # ARC job status retrieval on demand: # But we should _not_ update the status in the mRSL files, since # other MiG code might rely on finding only valid "MiG" states. if configuration.arc_clusters and \ job_dict.get('UNIQUE_RESOURCE_NAME', 'unset') == 'ARC' \ and job_dict['STATUS'] == 'EXECUTING': try: home = os.path.join(configuration.user_home, client_dir) arcsession = arcwrapper.Ui(home) arcstatus = arcsession.jobStatus(job_dict['EXE']) job_obj['status'] = arcstatus['status'] except arcwrapper.ARCWrapperError as err: logger.error('Error retrieving ARC job status: %s' % err.what()) job_obj['status'] += '(Error: ' + err.what() + ')' except arcwrapper.NoProxyError as err: logger.error('While retrieving ARC job status: %s' % err.what()) job_obj['status'] += '(Error: ' + err.what() + ')' except Exception as err: logger.error('Error retrieving ARC job status: %s' % err) job_obj['status'] += '(Error during retrieval)' exec_histories = [] if verbose(flags): if 'EXECUTE' in job_dict: command_line = '; '.join(job_dict['EXECUTE']) if len(command_line) > 256: job_obj['execute'] = '%s ...' % command_line[:252] else: job_obj['execute'] = command_line res_conf = job_dict.get('RESOURCE_CONFIG', {}) if 'RESOURCE_ID' in res_conf: public_id = res_conf['RESOURCE_ID'] if res_conf.get('ANONYMOUS', True): public_id = anon_resource_id(public_id) job_obj['resource'] = public_id if job_dict.get('PUBLICNAME', False): job_obj['resource'] += ' (alias %(PUBLICNAME)s)' % job_dict if 'RESOURCE_VGRID' in job_dict: job_obj['vgrid'] = job_dict['RESOURCE_VGRID'] if 'EXECUTION_HISTORY' in job_dict: counter = 0 for history_dict in job_dict['EXECUTION_HISTORY']: exec_history = \ {'object_type': 'execution_history'} if 'QUEUED_TIMESTAMP' in history_dict: exec_history['queued'] = \ time.asctime(history_dict['QUEUED_TIMESTAMP' ]) if 'EXECUTING_TIMESTAMP' in history_dict: exec_history['executing'] = \ time.asctime(history_dict['EXECUTING_TIMESTAMP' ]) if 'PUBLICNAME' in history_dict: if history_dict['PUBLICNAME']: exec_history['resource'] = \ history_dict['PUBLICNAME'] else: exec_history['resource'] = 'HIDDEN' if 'RESOURCE_VGRID' in history_dict: exec_history['vgrid'] = \ history_dict['RESOURCE_VGRID'] if 'FAILED_TIMESTAMP' in history_dict: exec_history['failed'] = \ time.asctime(history_dict['FAILED_TIMESTAMP' ]) if 'FAILED_MESSAGE' in history_dict: exec_history['failed_message'] = \ history_dict['FAILED_MESSAGE'] exec_histories.append( {'execution_history': exec_history, 'count': counter}) counter += 1 if 'SCHEDULE_HINT' in job_dict: job_obj['schedule_hint'] = job_dict['SCHEDULE_HINT'] # We should not show raw schedule_targets due to lack of anonymization if 'SCHEDULE_TARGETS' in job_dict: job_obj['schedule_hits'] = len(job_dict['SCHEDULE_TARGETS']) if 'EXPECTED_DELAY' in job_dict: # Catch None value if not job_dict['EXPECTED_DELAY']: job_obj['expected_delay'] = 0 else: job_obj['expected_delay'] = int(job_dict['EXPECTED_DELAY']) job_obj['execution_histories'] = exec_histories if interactive(flags): job_obj['statuslink'] = {'object_type': 'link', 'destination': 'fileman.py?path=%s/%s/' % (job_output_dir, job_id), 'text': 'View status files'} job_obj['mrsllink'] = {'object_type': 'link', 'destination': 'mrslview.py?job_id=%s' % job_id, 'text': 'View parsed mRSL contents'} if 'OUTPUTFILES' in job_dict and job_dict['OUTPUTFILES']: # Create a single ls link with all supplied outputfiles path_string = '' for path in job_dict['OUTPUTFILES']: # OUTPUTFILES is either just combo path or src dst paths parts = path.split() # Always take last part as destination path_string += 'path=%s;' % parts[-1] job_obj['outputfileslink'] = {'object_type': 'link', 'destination': 'ls.py?%s' % path_string, 'text': 'View output files'} form_method = 'post' csrf_limit = get_csrf_limit(configuration) target_op = 'resubmit' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) js_name = 'resubmit%s' % hexlify(job_id) helper = html_post_helper(js_name, '%s.py' % target_op, {'job_id': job_id, csrf_field: csrf_token}) output_objects.append({'object_type': 'html_form', 'text': helper}) job_obj['resubmitlink'] = {'object_type': 'link', 'destination': "javascript: %s();" % js_name, 'text': 'Resubmit job'} target_op = 'jobaction' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) js_name = 'freeze%s' % hexlify(job_id) helper = html_post_helper(js_name, '%s.py' % target_op, {'action': 'freeze', 'job_id': job_id, csrf_field: csrf_token}) output_objects.append({'object_type': 'html_form', 'text': helper}) job_obj['freezelink'] = {'object_type': 'link', 'destination': "javascript: %s();" % js_name, 'text': 'Freeze job in queue'} js_name = 'thaw%s' % hexlify(job_id) helper = html_post_helper(js_name, '%s.py' % target_op, {'action': 'thaw', 'job_id': job_id, csrf_field: csrf_token}) output_objects.append({'object_type': 'html_form', 'text': helper}) job_obj['thawlink'] = {'object_type': 'link', 'destination': "javascript: %s();" % js_name, 'text': 'Thaw job in queue'} js_name = 'cancel%s' % hexlify(job_id) helper = html_post_helper(js_name, '%s.py' % target_op, {'action': 'cancel', 'job_id': job_id, csrf_field: csrf_token}) output_objects.append({'object_type': 'html_form', 'text': helper}) job_obj['cancellink'] = {'object_type': 'link', 'destination': "javascript: %s();" % js_name, 'text': 'Cancel job'} target_op = 'jobschedule' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) js_name = 'jobschedule%s' % hexlify(job_id) helper = html_post_helper(js_name, '%s.py' % target_op, {'job_id': job_id, csrf_field: csrf_token}) output_objects.append({'object_type': 'html_form', 'text': helper}) job_obj['jobschedulelink'] = {'object_type': 'link', 'destination': "javascript: %s();" % js_name, 'text': 'Request schedule information'} target_op = 'jobfeasible' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) js_name = 'jobfeasible%s' % hexlify(job_id) helper = html_post_helper(js_name, '%s.py' % target_op, {'job_id': job_id, csrf_field: csrf_token}) output_objects.append({'object_type': 'html_form', 'text': helper}) job_obj['jobfeasiblelink'] = {'object_type': 'link', 'destination': "javascript: %s();" % js_name, 'text': 'Check job feasibility'} job_obj['liveiolink'] = {'object_type': 'link', 'destination': 'liveio.py?job_id=%s' % job_id, 'text': 'Request live I/O'} job_list['jobs'].append(job_obj) output_objects.append(job_list) return (output_objects, status)
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] status = returnvalues.OK (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: path can use wildcards, dst cannot typecheck_overrides={'path': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) patterns = accepted['path'] dst = accepted['dst'][-1].lstrip(os.sep) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) if dst: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) dst_mode = "wb" # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) relative_dst = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): logger.warning('%s tried to %s into restricted path %s ! (%s)' % (client_id, op_name, abs_dest, dst)) output_objects.append({ 'object_type': 'error_text', 'text': "invalid destination: '%s'" % dst }) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: output_lines = [] relative_path = abs_path.replace(base_dir, '') try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'accessed', [relative_path]) fd = open(abs_path, 'r') # use file directly as iterator for efficiency for line in fd: output_lines.append(line) fd.close() except Exception as exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'accessed', [relative_path], failed=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if dst: try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'modified', [dst]) out_fd = open(abs_dest, dst_mode) out_fd.writelines(output_lines) out_fd.close() logger.info('%s %s %s done' % (op_name, abs_path, abs_dest)) except Exception as exc: if not isinstance(exc, GDPIOLogError): gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'modified', [dst], failed=True, details=exc) output_objects.append({ 'object_type': 'error_text', 'text': "write failed: '%s'" % exc }) logger.error("%s: write failed on '%s': %s" % (op_name, abs_dest, exc)) status = returnvalues.SYSTEM_ERROR continue output_objects.append({ 'object_type': 'text', 'text': "wrote %s to %s" % (relative_path, relative_dst) }) # Prevent truncate after first write dst_mode = "ab+" else: entry = { 'object_type': 'file_output', 'lines': output_lines, 'wrap_binary': binary(flags), 'wrap_targets': ['lines'] } if verbose(flags): entry['path'] = relative_path output_objects.append(entry) # TODO: rip this hack out into real download handler? # Force download of files when output_format == 'file_format' # This will only work for the first file matching a glob when # using file_format. # And it is supposed to only work for one file. if 'output_format' in user_arguments_dict: output_format = user_arguments_dict['output_format'][0] if output_format == 'file': output_objects.append({ 'object_type': 'start', 'headers': [('Content-Disposition', 'attachment; filename="%s";' % os.path.basename(abs_path))] }) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'Delete frozen archive' defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: path cannot use wildcards here typecheck_overrides={}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flavor = accepted['flavor'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append( {'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not flavor in freeze_flavors.keys(): output_objects.append({'object_type': 'error_text', 'text': 'Invalid freeze flavor: %s' % flavor}) return (output_objects, returnvalues.CLIENT_ERROR) title = freeze_flavors[flavor]['deletefreeze_title'] output_objects.append({'object_type': 'header', 'text': title}) title_entry = find_entry(output_objects, 'title') title_entry['text'] = title if not configuration.site_enable_freeze: output_objects.append({'object_type': 'text', 'text': '''Freezing archives is disabled on this site. Please contact the site admins %s if you think it should be enabled. ''' % configuration.admin_email}) return (output_objects, returnvalues.OK) freeze_id = accepted['freeze_id'][-1] target = accepted['target'][-1] path_list = accepted['path'] if not target in valid_targets: output_objects.append({'object_type': 'error_text', 'text': 'Invalid delete freeze target: %s' % target}) return (output_objects, returnvalues.CLIENT_ERROR) # NB: the restrictions on freeze_id prevents illegal directory traversal if not is_frozen_archive(client_id, freeze_id, configuration): logger.error("%s: invalid freeze '%s': %s" % (op_name, client_id, freeze_id)) output_objects.append({'object_type': 'error_text', 'text': "No such frozen archive: '%s'" % freeze_id}) return (output_objects, returnvalues.CLIENT_ERROR) (load_status, freeze_dict) = get_frozen_archive(client_id, freeze_id, configuration, checksum_list=[]) if not load_status: logger.error("%s: load failed for '%s': %s" % (op_name, freeze_id, freeze_dict)) output_objects.append( {'object_type': 'error_text', 'text': 'Could not read frozen archive details for %s' % freeze_id}) return (output_objects, returnvalues.SYSTEM_ERROR) # Make sure the frozen archive belongs to the user trying to delete it if client_id != freeze_dict['CREATOR']: logger.error("%s: illegal access attempt for '%s': %s" % (op_name, freeze_id, client_id)) output_objects.append({'object_type': 'error_text', 'text': 'You are not the owner of frozen archive "%s"' % freeze_id}) return (output_objects, returnvalues.CLIENT_ERROR) if freeze_dict.get('FLAVOR', 'freeze') != flavor: logger.error("%s: flavor mismatch for '%s': %s vs %s" % (op_name, freeze_id, flavor, freeze_dict)) output_objects.append({'object_type': 'error_text', 'text': 'No such %s archive "%s"' % (flavor, freeze_id)}) return (output_objects, returnvalues.CLIENT_ERROR) # Prevent user-delete of the frozen archive if configuration forbids it. # We exclude any archives in the pending intermediate freeze state. # Freeze admins are also excluded from the restrictions. state = freeze_dict.get('STATE', keyword_final) if state == keyword_updating: output_objects.append( {'object_type': 'error_text', 'text': "Can't change %s archive %s which is currently being updated" % (flavor, freeze_id)}) output_objects.append({ 'object_type': 'link', 'destination': 'showfreeze.py?freeze_id=%s;flavor=%s' % (freeze_id, flavor), 'class': 'viewarchivelink iconspace genericbutton', 'title': 'View details about your %s archive' % flavor, 'text': 'View details', }) return (output_objects, returnvalues.CLIENT_ERROR) elif state == keyword_final and \ flavor in configuration.site_permanent_freeze and \ not client_id in configuration.site_freeze_admins: output_objects.append( {'object_type': 'error_text', 'text': "Can't change %s archives like '%s' yourself due to site policy" % (flavor, freeze_id)}) return (output_objects, returnvalues.CLIENT_ERROR) client_dir = client_id_dir(client_id) user_archives = os.path.join(configuration.freeze_home, client_dir) # Please note that base_dir must end in slash to avoid access to other # user archive dirs if own name is a prefix of another archive name base_dir = os.path.abspath(os.path.join(user_archives, freeze_id)) + os.sep if target == TARGET_ARCHIVE: # Delete the entire freeze archive (del_status, msg) = delete_frozen_archive(freeze_dict, client_id, configuration) # If something goes wrong when trying to delete freeze archive # freeze_id, an error is displayed. if not del_status: logger.error("%s: failed for '%s': %s" % (op_name, freeze_id, msg)) output_objects.append( {'object_type': 'error_text', 'text': 'Could not remove entire %s archive %s: %s' % (flavor, freeze_id, msg)}) return (output_objects, returnvalues.SYSTEM_ERROR) # If deletion of frozen archive freeze_id is successful, we just # return OK else: logger.info("%s: successful for '%s': %s" % (op_name, freeze_id, client_id)) output_objects.append( {'object_type': 'text', 'text': 'Successfully deleted %s archive: "%s"' % (flavor, freeze_id)}) elif target == TARGET_PATH: # Delete individual files in non-final archive del_paths = [] for path in path_list: # IMPORTANT: path must be expanded to abs for proper chrooting server_path = os.path.join(base_dir, path) abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, False): # Out of bounds! logger.warning('%s tried to %s del restricted path %s ! ( %s)' % (client_id, op_name, abs_path, path)) output_objects.append( {'object_type': 'error_text', 'text': 'Not allowed to delete %s - outside archive %s !' % (path, freeze_id)}) continue del_paths.append(path) (del_status, msg_list) = delete_archive_files(freeze_dict, client_id, del_paths, configuration) # If something goes wrong when trying to delete files from archive # freeze_id, an error is displayed. if not del_status: logger.error("%s: delete archive file(s) failed for '%s':\n%s" % (op_name, freeze_id, '\n'.join(msg_list))) output_objects.append( {'object_type': 'error_text', 'text': 'Could not remove file(s) from archive %s: %s' % (freeze_id, '\n '.join(msg_list))}) return (output_objects, returnvalues.SYSTEM_ERROR) # If deletion of files from archive freeze_id is successful, we just # return OK else: logger.info("%s: delete %d files successful for '%s': %s" % (op_name, len(path_list), freeze_id, client_id)) output_objects.append( {'object_type': 'text', 'text': 'Successfully deleted %d file(s) from archive: "%s"' % (len(path_list), freeze_id)}) # Success - show link to overview output_objects.append({'object_type': 'link', 'destination': 'freezedb.py', 'class': 'infolink iconspace', 'title': 'Show archives', 'text': 'Show archives'}) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False, op_menu=client_id) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input( user_arguments_dict, defaults, output_objects, allow_rejects=False, # NOTE: path can use wildcards typecheck_overrides={'path': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) pattern_list = accepted['path'] iosessionid = accepted['iosessionid'][-1] share_id = accepted['share_id'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Either authenticated user client_id set or sharelink ID if client_id: user_id = client_id target_dir = client_id_dir(client_id) base_dir = configuration.user_home id_query = '' page_title = 'Remove User File' if force(flags): rm_helper = delete_path else: rm_helper = remove_path userstyle = True widgets = True elif share_id: try: (share_mode, _) = extract_mode_id(configuration, share_id) except ValueError as err: logger.error('%s called with invalid share_id %s: %s' % (op_name, share_id, err)) output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid sharelink ID: %s' % share_id }) return (output_objects, returnvalues.CLIENT_ERROR) # TODO: load and check sharelink pickle (currently requires client_id) user_id = 'anonymous user through share ID %s' % share_id if share_mode == 'read-only': logger.error('%s called without write access: %s' % (op_name, accepted)) output_objects.append({ 'object_type': 'error_text', 'text': 'No write access!' }) return (output_objects, returnvalues.CLIENT_ERROR) target_dir = os.path.join(share_mode, share_id) base_dir = configuration.sharelink_home id_query = '?share_id=%s' % share_id page_title = 'Remove Shared File' rm_helper = delete_path userstyle = False widgets = False elif iosessionid.strip() and iosessionid.isalnum(): user_id = iosessionid base_dir = configuration.webserver_home target_dir = iosessionid page_title = 'Remove Session File' rm_helper = delete_path userstyle = False widgets = False else: logger.error('%s called without proper auth: %s' % (op_name, accepted)) output_objects.append({ 'object_type': 'error_text', 'text': 'Authentication is missing!' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(base_dir, target_dir)) + os.sep title_entry = find_entry(output_objects, 'title') title_entry['text'] = page_title title_entry['skipwidgets'] = not widgets title_entry['skipuserstyle'] = not userstyle output_objects.append({'object_type': 'header', 'text': page_title}) logger.debug("%s: with paths: %s" % (op_name, pattern_list)) # Input validation assures target_dir can't escape base_dir if not os.path.isdir(base_dir): output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid client/sharelink/session id!' }) logger.warning('%s used %s with invalid base dir: %s' % (user_id, op_name, base_dir)) return (output_objects, returnvalues.CLIENT_ERROR) if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! ( %s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: logger.warning("%s: no matching paths: %s" % (op_name, pattern_list)) output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: real_path = os.path.realpath(abs_path) relative_path = abs_path.replace(base_dir, '') if verbose(flags): output_objects.append({ 'object_type': 'file', 'name': relative_path }) # Make it harder to accidentially delete too much - e.g. do not # delete VGrid files without explicit selection of subdir contents if abs_path == os.path.abspath(base_dir): logger.error("%s: refusing rm home dir: %s" % (op_name, abs_path)) output_objects.append({ 'object_type': 'warning', 'text': "You're not allowed to delete your entire home directory!" }) status = returnvalues.CLIENT_ERROR continue # Generally refuse handling symlinks including root vgrid shares elif os.path.islink(abs_path): logger.error("%s: refusing rm link: %s" % (op_name, abs_path)) output_objects.append({ 'object_type': 'warning', 'text': """ You're not allowed to delete entire special folders like %s shares and %s """ % (configuration.site_vgrid_label, trash_linkname) }) status = returnvalues.CLIENT_ERROR continue # Additionally refuse operations on inherited subvgrid share roots elif in_vgrid_share(configuration, abs_path) == relative_path: output_objects.append({ 'object_type': 'warning', 'text': """You're not allowed to remove entire %s shared folders!""" % configuration.site_vgrid_label }) status = returnvalues.CLIENT_ERROR continue elif os.path.isdir(abs_path) and not recursive(flags): logger.error("%s: non-recursive call on dir '%s'" % (op_name, abs_path)) output_objects.append({ 'object_type': 'error_text', 'text': "cannot remove '%s': is a direcory" % relative_path }) status = returnvalues.CLIENT_ERROR continue trash_base = get_trash_location(configuration, abs_path) if not trash_base and not force(flags): logger.error("%s: no trash for dir '%s'" % (op_name, abs_path)) output_objects.append({ 'object_type': 'error_text', 'text': "No trash enabled for '%s' - read-only?" % relative_path }) status = returnvalues.CLIENT_ERROR continue try: if rm_helper == remove_path and \ os.path.commonprefix([real_path, trash_base]) \ == trash_base: logger.warning("%s: already in trash: '%s'" % (op_name, real_path)) output_objects.append({ 'object_type': 'error_text', 'text': """ '%s' is already in trash - no action: use force flag to permanently delete""" % relative_path }) status = returnvalues.CLIENT_ERROR continue except Exception as err: logger.error("%s: check trash failed: %s" % (op_name, err)) continue if not check_write_access(abs_path): logger.warning('%s called without write access: %s' % (op_name, abs_path)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot remove "%s": inside a read-only location!' % pattern }) status = returnvalues.CLIENT_ERROR continue # TODO: limit delete in vgrid share trash to vgrid owners / conf? # ... malicious members can still e.g. truncate all files. # we could consider removing write bit on move to trash. # TODO: user setting to switch on/off trash? # TODO: add direct delete checkbox in fileman move to trash dialog? # TODO: add empty trash option for Trash? # TODO: user settings to define read-only and auto-expire in trash? # TODO: add trash support for sftp/ftps/webdavs? gdp_iolog_action = 'deleted' gdp_iolog_paths = [relative_path] if rm_helper == remove_path: gdp_iolog_action = 'moved' trash_base_path = \ get_trash_location(configuration, abs_path, True) trash_relative_path = \ trash_base_path.replace(configuration.user_home, '') trash_relative_path = \ trash_relative_path.replace( configuration.vgrid_files_home, '') gdp_iolog_paths.append(trash_relative_path) try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], gdp_iolog_action, gdp_iolog_paths) gdp_iolog_status = True except GDPIOLogError as exc: gdp_iolog_status = False rm_err = [str(exc)] rm_status = False if gdp_iolog_status: (rm_status, rm_err) = rm_helper(configuration, abs_path) if not rm_status or not gdp_iolog_status: if gdp_iolog_status: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], gdp_iolog_action, gdp_iolog_paths, failed=True, details=rm_err) logger.error("%s: failed on '%s': %s" % (op_name, abs_path, ', '.join(rm_err))) output_objects.append({ 'object_type': 'error_text', 'text': "remove '%s' failed: %s" % (relative_path, '. '.join(rm_err)) }) status = returnvalues.SYSTEM_ERROR continue logger.info("%s: successfully (re)moved %s" % (op_name, abs_path)) output_objects.append({ 'object_type': 'text', 'text': "removed %s" % (relative_path) }) output_objects.append({ 'object_type': 'link', 'destination': 'ls.py%s' % id_query, 'text': 'Return to files overview' }) return (output_objects, status)
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False, op_menu=client_id) defaults = signature()[1] (validate_status, accepted) = validate_input( user_arguments_dict, defaults, output_objects, allow_rejects=False, # NOTE: path can use wildcards, current_dir cannot typecheck_overrides={'path': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) pattern_list = accepted['path'] current_dir = accepted['current_dir'][-1].lstrip('/') share_id = accepted['share_id'][-1] status = returnvalues.OK read_mode, write_mode = True, True # Either authenticated user client_id set or sharelink ID if client_id: user_id = client_id target_dir = client_id_dir(client_id) base_dir = configuration.user_home redirect_name = configuration.site_user_redirect redirect_path = redirect_name id_args = '' root_link_name = 'USER HOME' main_class = "user_ls" page_title = 'User Files' userstyle = True widgets = True visibility_mods = ''' .%(main_class)s .disable_read { display: none; } .%(main_class)s .disable_write { display: none; } ''' elif share_id: try: (share_mode, _) = extract_mode_id(configuration, share_id) except ValueError as err: logger.error('%s called with invalid share_id %s: %s' % (op_name, share_id, err)) output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid sharelink ID: %s' % share_id }) return (output_objects, returnvalues.CLIENT_ERROR) # TODO: load and check sharelink pickle (currently requires client_id) # then include shared by %(owner)s on page header user_id = 'anonymous user through share ID %s' % share_id target_dir = os.path.join(share_mode, share_id) base_dir = configuration.sharelink_home redirect_name = 'share_redirect' redirect_path = os.path.join(redirect_name, share_id) id_args = 'share_id=%s;' % share_id root_link_name = '%s' % share_id main_class = "sharelink_ls" page_title = 'Shared Files' userstyle = False widgets = False # default to include file info if flags == '': flags += 'f' if share_mode == 'read-only': write_mode = False visibility_mods = ''' .%(main_class)s .enable_write { display: none; } .%(main_class)s .disable_read { display: none; } ''' elif share_mode == 'write-only': read_mode = False visibility_mods = ''' .%(main_class)s .enable_read { display: none; } .%(main_class)s .disable_write { display: none; } ''' else: visibility_mods = ''' .%(main_class)s .disable_read { display: none; } .%(main_class)s .disable_write { display: none; } ''' else: logger.error('%s called without proper auth: %s' % (op_name, accepted)) output_objects.append({ 'object_type': 'error_text', 'text': 'Authentication is missing!' }) return (output_objects, returnvalues.SYSTEM_ERROR) visibility_toggle = ''' <style> %s </style> ''' % (visibility_mods % { 'main_class': main_class }) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(base_dir, target_dir)) + os.sep if not os.path.isdir(base_dir): logger.error('%s called on missing base_dir: %s' % (op_name, base_dir)) output_objects.append({ 'object_type': 'error_text', 'text': 'No such %s!' % page_title.lower() }) return (output_objects, returnvalues.CLIENT_ERROR) title_entry = find_entry(output_objects, 'title') title_entry['text'] = page_title title_entry['skipwidgets'] = not widgets title_entry['skipuserstyle'] = not userstyle user_settings = title_entry.get('user_settings', {}) open_button_id = 'open_fancy_upload' form_method = 'post' csrf_limit = get_csrf_limit(configuration) fill_helpers = { 'dest_dir': current_dir + os.sep, 'share_id': share_id, 'flags': flags, 'tmp_flags': flags, 'long_set': long_list(flags), 'recursive_set': recursive(flags), 'all_set': all(flags), 'fancy_open': open_button_id, 'fancy_dialog': fancy_upload_html(configuration), 'form_method': form_method, 'csrf_field': csrf_field, 'csrf_limit': csrf_limit } target_op = 'uploadchunked' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token}) (cf_import, cf_init, cf_ready) = confirm_js(configuration) (fu_import, fu_init, fu_ready) = fancy_upload_js(configuration, 'function() { location.reload(); }', share_id, csrf_token) add_import = ''' %s %s ''' % (cf_import, fu_import) add_init = ''' %s %s %s %s ''' % (cf_init, fu_init, select_all_javascript(), selected_file_actions_javascript()) add_ready = ''' %s %s /* wrap openFancyUpload in function to avoid event data as argument */ $("#%s").click(function() { openFancyUpload(); }); $("#checkall_box").click(toggleChecked); ''' % (cf_ready, fu_ready, open_button_id) # TODO: can we update style inline to avoid explicit themed_styles? styles = themed_styles( configuration, advanced=['jquery.fileupload.css', 'jquery.fileupload-ui.css'], skin=['fileupload-ui.custom.css'], user_settings=user_settings) styles['advanced'] += ''' %s ''' % visibility_toggle title_entry['style'] = styles title_entry['script']['advanced'] += add_import title_entry['script']['init'] += add_init title_entry['script']['ready'] += add_ready title_entry['script']['body'] = ' class="%s"' % main_class output_objects.append({'object_type': 'header', 'text': page_title}) # TODO: move to output html handler output_objects.append({ 'object_type': 'html_form', 'text': confirm_html(configuration) }) # Shared URL helpers ls_url_template = 'ls.py?%scurrent_dir=%%(rel_dir_enc)s;flags=%s' % \ (id_args, flags) csrf_token = make_csrf_token(configuration, form_method, 'rm', client_id, csrf_limit) rm_url_template = 'rm.py?%spath=%%(rel_path_enc)s;%s=%s' % \ (id_args, csrf_field, csrf_token) rmdir_url_template = 'rm.py?%spath=%%(rel_path_enc)s;flags=r;%s=%s' % \ (id_args, csrf_field, csrf_token) editor_url_template = 'editor.py?%spath=%%(rel_path_enc)s' % id_args redirect_url_template = '/%s/%%(rel_path_enc)s' % redirect_path location_pre_html = """ <div class='files'> <table class='files'> <tr class=title><td class=centertext> Working directory: </td></tr> <tr><td class='centertext'> """ output_objects.append({ 'object_type': 'html_form', 'text': location_pre_html }) # Use current_dir nav location links for pattern in pattern_list[:1]: links = [] links.append({ 'object_type': 'link', 'text': root_link_name, 'destination': ls_url_template % { 'rel_dir_enc': '.' } }) prefix = '' parts = os.path.normpath(current_dir).split(os.sep) for i in parts: if i == ".": continue prefix = os.path.join(prefix, i) links.append({ 'object_type': 'link', 'text': i, 'destination': ls_url_template % { 'rel_dir_enc': quote(prefix) } }) output_objects.append({ 'object_type': 'multilinkline', 'links': links, 'sep': ' %s ' % os.sep }) location_post_html = """ </td></tr> </table> </div> <br /> """ output_objects.append({ 'object_type': 'html_form', 'text': location_post_html }) more_html = """ <div class='files if_full'> <form method='%(form_method)s' name='fileform' onSubmit='return selectedFilesAction();'> <table class='files'> <tr class=title><td class=centertext colspan=2> Advanced file actions </td></tr> <tr><td> Action on paths selected below (please hold mouse cursor over button for a description): </td> <td class=centertext> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='v' /> <input type='submit' title='Show concatenated contents (cat)' onClick='document.pressed=this.value' value='cat' /> <input type='submit' onClick='document.pressed=this.value' value='head' title='Show first lines (head)' /> <input type='submit' onClick='document.pressed=this.value' value='tail' title='Show last lines (tail)' /> <input type='submit' onClick='document.pressed=this.value' value='wc' title='Count lines/words/chars (wc)' /> <input type='submit' onClick='document.pressed=this.value' value='stat' title='Show details (stat)' /> <input type='submit' onClick='document.pressed=this.value' value='touch' title='Update timestamp (touch)' /> <input type='submit' onClick='document.pressed=this.value' value='truncate' title='truncate! (truncate)' /> <input type='submit' onClick='document.pressed=this.value' value='rm' title='delete! (rm)' /> <input type='submit' onClick='document.pressed=this.value' value='rmdir' title='Remove directory (rmdir)' /> <input type='submit' onClick='document.pressed=this.value' value='submit' title='Submit file (submit)' /> </td></tr> </table> </form> </div> """ % { 'form_method': form_method } output_objects.append({'object_type': 'html_form', 'text': more_html}) dir_listings = [] output_objects.append({ 'object_type': 'dir_listings', 'dir_listings': dir_listings, 'flags': flags, 'redirect_name': redirect_name, 'redirect_path': redirect_path, 'share_id': share_id, 'ls_url_template': ls_url_template, 'rm_url_template': rm_url_template, 'rmdir_url_template': rmdir_url_template, 'editor_url_template': editor_url_template, 'redirect_url_template': redirect_url_template, }) first_match = None for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages current_path = os.path.normpath(os.path.join(base_dir, current_dir)) unfiltered_match = glob.glob(current_path + os.sep + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning('%s tried to %s restricted path %s ! (%s)' % (user_id, op_name, abs_path, pattern)) continue match.append(abs_path) if not first_match: first_match = abs_path # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND # Never show any ls output in write-only mode (css hide is not enough!) if not read_mode: continue for abs_path in match: if abs_path + os.sep == base_dir: relative_path = '.' else: relative_path = abs_path.replace(base_dir, '') entries = [] dir_listing = { 'object_type': 'dir_listing', 'relative_path': relative_path, 'entries': entries, 'flags': flags, } try: gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'], 'accessed', [relative_path]) except GDPIOLogError as exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) continue handle_ls(configuration, output_objects, entries, base_dir, abs_path, flags, 0) dir_listings.append(dir_listing) output_objects.append({ 'object_type': 'html_form', 'text': """<br/> <div class='files disable_read'> <p class='info icon'>""" }) # Shared message for text (e.g. user scripts) and html-format if not read_mode: # Please note that we use verbatim to get info icon right in html output_objects.append({ 'object_type': 'verbatim', 'text': """ This is a write-only share so you do not have access to see the files, only upload data and create directories. """ }) output_objects.append({ 'object_type': 'html_form', 'text': """ </p> </div> <div class='files enable_read'> <form method='get' action='ls.py'> <table class='files'> <tr class=title><td class=centertext> Filter paths (wildcards like * and ? are allowed) <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> <input type='text' name='path' value='' /> <input type='submit' value='Filter' /> </td></tr> </table> </form> </div> """ % fill_helpers }) # Short/long format buttons fill_helpers['tmp_flags'] = flags + 'l' htmlform = """ <table class='files if_full'> <tr class=title><td class=centertext colspan=4> File view options </td></tr> <tr><td colspan=4><br /></td></tr> <tr class=title><td>Parameter</td><td>Setting</td><td>Enable</td><td>Disable</td></tr> <tr><td>Long format</td><td> %(long_set)s</td><td> <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />" % entry fill_helpers['tmp_flags'] = flags.replace('l', '') htmlform += """ <input type='submit' value='On' /><br /> </form> </td><td> <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />" % entry htmlform += """ <input type='submit' value='Off' /><br /> </form> </td></tr> """ # Recursive output fill_helpers['tmp_flags'] = flags + 'r' htmlform += """ <!-- Non-/recursive list buttons --> <tr><td>Recursion</td><td> %(recursive_set)s</td><td>""" % fill_helpers htmlform += """ <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />" % entry fill_helpers['tmp_flags'] = flags.replace('r', '') htmlform += """ <input type='submit' value='On' /><br /> </form> </td><td> <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />"\ % entry htmlform += """ <input type='submit' value='Off' /><br /> </form> </td></tr> """ htmlform += """ <!-- Show dot files buttons --> <tr><td>Show hidden files</td><td> %(all_set)s</td><td>""" % fill_helpers fill_helpers['tmp_flags'] = flags + 'a' htmlform += """ <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />" % entry fill_helpers['tmp_flags'] = flags.replace('a', '') htmlform += """ <input type='submit' value='On' /><br /> </form> </td><td> <form method='get' action='ls.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='flags' value='%(tmp_flags)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> """ % fill_helpers for entry in pattern_list: htmlform += "<input type='hidden' name='path' value='%s' />" % entry htmlform += """ <input type='submit' value='Off' /><br /> </form> </td></tr> </table> """ # show flag buttons after contents to limit clutter output_objects.append({'object_type': 'html_form', 'text': htmlform}) # create additional action forms if first_match: htmlform = """ <br /> <div class='files disable_write'> <p class='info icon'> This is a read-only share so you do not have access to edit or add files, only view data. </p> </div> <table class='files enable_write if_full'> <tr class=title><td class=centertext colspan=3> Edit file </td></tr> <tr><td> Fill in the path of a file to edit and press 'edit' to open that file in the<br /> online file editor. Alternatively a file can be selected for editing through<br /> the listing of personal files. </td><td colspan=2 class=righttext> <form name='editor' method='get' action='editor.py'> <input type='hidden' name='output_format' value='html' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> <input type='text' name='path' size=50 value='' required /> <input type='submit' value='edit' /> </form> </td></tr> </table> <br />""" % fill_helpers target_op = 'mkdir' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token}) htmlform += """ <table class='files enable_write'> <tr class=title><td class=centertext colspan=4> Create directory </td></tr> <tr><td> Name of new directory to be created in current directory (%(dest_dir)s) </td><td class=righttext colspan=3> <form method='%(form_method)s' action='%(target_op)s.py'> <input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <input name='current_dir' type='hidden' value='%(dest_dir)s' /> <input name='path' size=50 required /> <input type='submit' value='Create' name='mkdirbutton' /> </form> </td></tr> </table> <br /> """ % fill_helpers target_op = 'textarea' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token}) htmlform += """ <form enctype='multipart/form-data' method='%(form_method)s' action='%(target_op)s.py'> <input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' /> <input type='hidden' name='share_id' value='%(share_id)s' /> <table class='files enable_write if_full'> <tr class='title'><td class=centertext colspan=4> Upload file </td></tr> <tr><td colspan=4> Upload file to current directory (%(dest_dir)s) </td></tr> <tr class='if_full'><td colspan=2> Extract package files (.zip, .tar.gz, .tar.bz2) </td><td colspan=2> <input type=checkbox name='extract_0' /> </td></tr> <tr class='if_full'><td colspan=2> Submit mRSL files (also .mRSL files included in packages) </td><td colspan=2> <input type=checkbox name='submitmrsl_0' /> </td></tr> <tr><td> File to upload </td><td class=righttext colspan=3> <input name='fileupload_0_0_0' type='file'/> </td></tr> <tr><td> Optional remote filename (extra useful in windows) </td><td class=righttext colspan=3> <input name='default_remotefilename_0' type='hidden' value='%(dest_dir)s'/> <input name='remotefilename_0' type='text' size='50' value='%(dest_dir)s'/> <input type='submit' value='Upload' name='sendfile'/> </td></tr> </table> </form> %(fancy_dialog)s <table class='files enable_write'> <tr class='title'><td class='centertext'> Upload files efficiently (using chunking). </td></tr> <tr><td class='centertext'> <button id='%(fancy_open)s'>Open Upload dialog</button> </td></tr> </table> <script type='text/javascript' > setUploadDest('%(dest_dir)s'); </script> """ % fill_helpers output_objects.append({'object_type': 'html_form', 'text': htmlform}) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) patterns = accepted['job_id'] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep status = returnvalues.OK filelist = [] for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to # avoid leaking information about file system layout while # allowing consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match^I if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % pattern }) status = returnvalues.CLIENT_ERROR else: filelist += match # job feasibility is hard on the server, limit if len(filelist) > 100: output_objects.append({ 'object_type': 'error_text', 'text': 'Too many matching jobs (%s)!' % len(filelist) }) return (output_objects, returnvalues.CLIENT_ERROR) checkcondjobs = [] for filepath in filelist: # Extract job_id from filepath (replace doesn't modify filepath) mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') checkcondjob = {'object_type': 'checkcondjob', 'job_id': job_id} dict = unpickle(filepath, logger) if not dict: checkcondjob['message'] = \ ('The file containing the information ' \ 'for job id %s could not be opened! ' \ 'You can only check feasibility of ' \ 'your own jobs!' ) % job_id checkcondjobs.append(checkcondjob) status = returnvalues.CLIENT_ERROR continue # Is the job status pending? possible_check_states = ['QUEUED', 'RETRY', 'FROZEN'] if not dict['STATUS'] in possible_check_states: checkcondjob['message'] = \ 'You can only check feasibility of jobs with status: %s.'\ % ' or '.join(possible_check_states) checkcondjobs.append(checkcondjob) continue # Actually check feasibility feasible_res = job_feasibility(configuration, dict) checkcondjob.update(feasible_res) checkcondjobs.append(checkcondjob) output_objects.append({ 'object_type': 'checkcondjobs', 'checkcondjobs': checkcondjobs }) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False, op_menu=client_id) # NOTE: we expose editor e.g. on sharelinks (ls) but do not allow it, yet if not client_id: output_objects.append({ 'object_type': 'error_text', 'text': 'The inline editor is currently only available for authenticated ' 'users on this site' }) return (output_objects, returnvalues.CLIENT_ERROR) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: path cannot use wildcards here typecheck_overrides={}, ) # TODO: if validator is too tight we should accept rejects here # and then make sure that such rejected fields are never printed if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) path = accepted['path'][-1] current_dir = accepted['current_dir'][-1] # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep # the client can choose to specify the path of the target directory with # current_dir + "/" + path, instead of specifying the complete path in # subdirs. This is usefull from ls.py where a hidden html control makes it # possible to target the directory from the current dir. title_entry = find_entry(output_objects, 'title') title_entry['text'] = '%s file web editor' % configuration.short_title # TODO: fully update to new style and script structure title_entry['style']['advanced'] += advanced_editor_css_deps() title_entry['script']['advanced'] += advanced_editor_js_deps() + '\n' title_entry['script']['advanced'] += lock_info('this file', -1) output_objects.append({ 'object_type': 'header', 'text': 'Editing file in %s home directory' % configuration.short_title }) if not path: now = time.gmtime() path = 'noname-%s.txt' % time.strftime('%d%m%y-%H%M%S', now) output_objects.append({ 'object_type': 'text', 'text': 'No path supplied - creating new file in %s' % path }) rel_path = os.path.join(current_dir.lstrip(os.sep), path.lstrip(os.sep)) # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(os.path.join(base_dir, rel_path)) if not valid_user_path(configuration, abs_path, base_dir): logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, rel_path)) output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % path }) return (output_objects, returnvalues.CLIENT_ERROR) (owner, time_left) = acquire_edit_lock(abs_path, client_id) if owner == client_id: javascript = \ '''<script type="text/javascript"> setTimeout("newcountdown('%s', %d)", 1) </script> '''\ % (path, time_left / 60) output_objects.append({'object_type': 'html_form', 'text': javascript}) html = edit_file(configuration, client_id, path, abs_path) output_objects.append({'object_type': 'html_form', 'text': html}) else: output_objects.append({ 'object_type': 'error_text', 'text': '%s acquired the editing lock for %s! (timeout in %d seconds)' % (owner, path, time_left) }) return (output_objects, returnvalues.CLIENT_ERROR) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: path can use wildcards, dst and current_dir cannot typecheck_overrides={'path': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) algo_list = accepted['hash_algo'] max_chunks = int(accepted['max_chunks'][-1]) pattern_list = accepted['path'] dst = accepted['dst'][-1] current_dir = accepted['current_dir'][-1].lstrip(os.sep) # All paths are relative to current_dir pattern_list = [os.path.join(current_dir, i) for i in pattern_list] if dst: dst = os.path.join(current_dir, dst) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dir = os.path.abspath( os.path.join(base_dir, current_dir.lstrip(os.sep))) if not valid_user_path(configuration, abs_dir, base_dir, True): output_objects.append({ 'object_type': 'error_text', 'text': "You're not allowed to work in %s!" % current_dir }) logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dir, current_dir)) return (output_objects, returnvalues.CLIENT_ERROR) if verbose(flags): output_objects.append({ 'object_type': 'text', 'text': "working in %s" % current_dir }) if dst: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # NOTE: dst already incorporates current_dir prefix here # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) logger.info('chksum in %s' % abs_dest) # Don't use abs_path in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % dst }) logger.warning('%s tried to %s restricted path %s !(%s)' % (client_id, op_name, abs_dest, dst)) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_dest)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot checksum to "%s": inside a read-only location!' % relative_dest }) return (output_objects, returnvalues.CLIENT_ERROR) all_lines = [] for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND for abs_path in match: relative_path = abs_path.replace(base_dir, '') output_lines = [] for hash_algo in algo_list: try: chksum_helper = _algo_map.get(hash_algo, _algo_map["md5"]) checksum = chksum_helper(abs_path, max_chunks=max_chunks) line = "%s %s\n" % (checksum, relative_path) logger.info("%s %s of %s: %s" % (op_name, hash_algo, abs_path, checksum)) output_lines.append(line) except Exception as exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue entry = {'object_type': 'file_output', 'lines': output_lines} output_objects.append(entry) all_lines += output_lines if dst and not write_file(''.join(all_lines), abs_dest, logger): output_objects.append({ 'object_type': 'error_text', 'text': "failed to write checksums to %s" % relative_dest }) logger.error("writing checksums to %s for %s failed" % (abs_dest, client_id)) status = returnvalues.SYSTEM_ERROR return (output_objects, status)
def main(client_id, user_arguments_dict, environ=None): """Main function used by front end""" if environ is None: environ = os.environ (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False, op_menu=client_id) # TODO: this handler is incomplete and NOT yet hooked up with Xgi-bin return (output_objects, returnvalues.SYSTEM_ERROR) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] (validate_status, accepted) = validate_input( user_arguments_dict, defaults, output_objects, allow_rejects=False, # NOTE: path cannot use wildcards here typecheck_overrides={}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) filename = accepted['filename'][-1] patterns = accepted['path'] iosessionid = accepted['iosessionid'][-1] file_startpos = accepted['file_startpos'][-1] file_endpos = accepted['file_endpos'][-1] if file_startpos: file_startpos = int(file_startpos) else: file_startpos = -1 if file_endpos: file_endpos = int(file_endpos) else: file_endpos = -1 # Legacy naming if filename: patterns = [filename] valid_methods = ['GET', 'PUT', 'DELETE'] action = os.getenv('REQUEST_METHOD') if not action in valid_methods: output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting %s requests''' % ', '.join(valid_methods) }) return (output_objects, returnvalues.CLIENT_ERROR) # TODO: introduce CSRF protection here when clients support it # if not safe_handler(configuration, action, op_name, client_id, # get_csrf_limit(configuration), accepted): # output_objects.append( # {'object_type': 'error_text', 'text': # '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' # }) # return (output_objects, returnvalues.CLIENT_ERROR) # Either authenticated user client_id set or job IO session ID if client_id: user_id = client_id target_dir = client_id_dir(client_id) base_dir = configuration.user_home page_title = 'User range file access' widgets = True userstyle = True elif iosessionid: user_id = iosessionid target_dir = iosessionid base_dir = configuration.webserver_home page_title = 'Create Shared Directory' widgets = False userstyle = False else: logger.error('%s called without proper auth: %s' % (op_name, accepted)) output_objects.append({ 'object_type': 'error_text', 'text': 'Authentication is missing!' }) return (output_objects, returnvalues.SYSTEM_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(base_dir, target_dir)) + os.sep title_entry = find_entry(output_objects, 'title') title_entry['text'] = page_title title_entry['skipwidgets'] = not widgets title_entry['skipuserstyle'] = not userstyle output_objects.append({'object_type': 'header', 'text': page_title}) # Input validation assures target_dir can't escape base_dir if not os.path.isdir(base_dir): output_objects.append({ 'object_type': 'error_text', 'text': 'Invalid client/iosession id!' }) return (output_objects, returnvalues.CLIENT_ERROR) for pattern in patterns: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages # NB: Globbing disabled on purpose here unfiltered_match = [base_dir + os.sep + pattern] match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warn('%s tried to %s %s restricted path! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'error_text', 'text': "%s: cannot access file '%s': Permission denied" % (op_name, pattern) }) status = returnvalues.CLIENT_ERROR for abs_path in match: relative_path = abs_path.replace(base_dir, '') if action == 'GET': if not do_get(configuration, output_objects, abs_path, file_startpos, file_endpos): output_objects.append({ 'object_type': 'error_text', 'text': '''Could not gett %r''' % pattern }) status = returnvalues.SYSTEM_ERROR elif action == 'PUT': if not do_put(configuration, output_objects, abs_path, file_startpos, file_endpos): output_objects.append({ 'object_type': 'error_text', 'text': '''Could not put %r''' % pattern }) status = returnvalues.SYSTEM_ERROR elif action == 'DELETE': if not do_delete(configuration, output_objects, abs_path): output_objects.append({ 'object_type': 'error_text', 'text': '''Could not delete %r''' % pattern }) status = returnvalues.SYSTEM_ERROR else: output_objects.append({ 'object_type': 'error_text', 'text': 'Unsupported action: %r' % action }) return (output_objects, returnvalues.CLIENT_ERROR) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) patterns = accepted['job_id'] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not configuration.site_enable_jobs: output_objects.append({ 'object_type': 'error_text', 'text': '''Job execution is not enabled on this system''' }) return (output_objects, returnvalues.SYSTEM_ERROR) if not patterns: output_objects.append({ 'object_type': 'error_text', 'text': 'No job_id specified!' }) return (output_objects, returnvalues.NO_SUCH_JOB_ID) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = \ os.path.abspath(os.path.join(configuration.mrsl_files_dir, client_dir)) + os.sep filelist = [] keywords_dict = mrslkeywords.get_keywords_dict(configuration) for pattern in patterns: pattern = pattern.strip() # Backward compatibility - all_jobs keyword should match all jobs if pattern == all_jobs: pattern = '*' # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern + '.mRSL') match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue # Insert valid job files in filelist for later treatment match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % pattern }) status = returnvalues.CLIENT_ERROR else: filelist += match # resubmit is hard on the server if len(filelist) > 100: output_objects.append({ 'object_type': 'error_text', 'text': 'Too many matching jobs (%s)!' % len(filelist) }) return (output_objects, returnvalues.CLIENT_ERROR) resubmitobjs = [] status = returnvalues.OK for filepath in filelist: mrsl_file = filepath.replace(base_dir, '') job_id = mrsl_file.replace('.mRSL', '') # ("Resubmitting job with job_id: %s" % job_id) resubmitobj = {'object_type': 'resubmitobj', 'job_id': job_id} mrsl_dict = unpickle(filepath, logger) if not mrsl_dict: resubmitobj['message'] = "No such job: %s (%s)" % (job_id, mrsl_file) status = returnvalues.CLIENT_ERROR resubmitobjs.append(resubmitobj) continue resubmit_items = keywords_dict.keys() # loop selected keywords and create mRSL string resubmit_job_string = '' for dict_elem in resubmit_items: value = '' # Extract job value with fallback to default to support optional # fields job_value = mrsl_dict.get(dict_elem, keywords_dict[dict_elem]['Value']) if keywords_dict[dict_elem]['Type'].startswith( 'multiplekeyvalues'): for (elem_key, elem_val) in job_value: if elem_key: value += '%s=%s\n' % (str(elem_key).strip(), str(elem_val).strip()) elif keywords_dict[dict_elem]['Type'].startswith('multiple'): for elem in job_value: if elem: value += '%s\n' % str(elem).rstrip() else: if str(job_value): value += '%s\n' % str(job_value).rstrip() # Only insert keywords with an associated value if value: if value.rstrip() != '': resubmit_job_string += '''::%s:: %s ''' % (dict_elem, value.rstrip()) # save tempfile (filehandle, tempfilename) = \ tempfile.mkstemp(dir=configuration.mig_system_files, text=True) os.write(filehandle, resubmit_job_string) os.close(filehandle) # submit job the usual way (new_job_status, msg, new_job_id) = new_job(tempfilename, client_id, configuration, False, True) if not new_job_status: resubmitobj['status'] = False resubmitobj['message'] = msg status = returnvalues.SYSTEM_ERROR resubmitobjs.append(resubmitobj) continue # o.out("Resubmit failed: %s" % msg) # o.reply_and_exit(o.ERROR) resubmitobj['status'] = True resubmitobj['new_job_id'] = new_job_id resubmitobjs.append(resubmitobj) # o.out("Resubmit successful: %s" % msg) # o.out("%s" % msg) output_objects.append({ 'object_type': 'resubmitobjs', 'resubmitobjs': resubmitobjs }) return (output_objects, status)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) client_dir = client_id_dir(client_id) defaults = signature()[1] title_entry = find_entry(output_objects, 'title') label = "%s" % configuration.site_vgrid_label title_entry['text'] = "Add/Update %s Trigger" % label output_objects.append({ 'object_type': 'header', 'text': 'Add/Update %s Trigger' % label }) (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, typecheck_overrides={'path': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) # NOTE: strip leftmost slashes from all fields used in file paths to avoid # interference with os.path.join calls. Furthermore we strip and normalize # the path variable first to make sure it does not point outside the vgrid. # In practice any such directory traversal attempts will generally be moot # since the grid_events daemon only starts a listener for each top-level # vgrid and in there only reacts to events that match trigger rules from # that particular vgrid. Thus only subvgrid access to parent vgrids might # be a concern and still of limited consequence. # NOTE: merge multi args into one string and split again to get flat array rule_id = accepted['rule_id'][-1].strip() vgrid_name = accepted['vgrid_name'][-1].strip().lstrip(os.sep) path = os.path.normpath(accepted['path'][-1].strip()).lstrip(os.sep) changes = [i.strip() for i in ' '.join(accepted['changes']).split()] action = accepted['action'][-1].strip() arguments = [ i.strip() for i in shlex.split(' '.join(accepted['arguments'])) ] rate_limit = accepted['rate_limit'][-1].strip() settle_time = accepted['settle_time'][-1].strip() match_files = accepted['match_files'][-1].strip() == 'True' match_dirs = accepted['match_dirs'][-1].strip() == 'True' match_recursive = accepted['match_recursive'][-1].strip() == 'True' rank_str = accepted['rank'][-1] try: rank = int(rank_str) except ValueError: rank = None logger.debug("addvgridtrigger with args: %s" % user_arguments_dict) if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep # we just use a high res timestamp as automatic rule_id if rule_id == keyword_auto: rule_id = "%d" % (time.time() * 1E8) if action == keyword_auto: action = valid_trigger_actions[0] if any_state in changes: changes = valid_trigger_changes logger.info("addvgridtrigger %s" % vgrid_name) # Validity of user and vgrid names is checked in this init function so # no need to worry about illegal directory traversal through variables (ret_val, msg, ret_variables) = \ init_vgrid_script_add_rem(vgrid_name, client_id, rule_id, 'trigger', configuration) if not ret_val: output_objects.append({'object_type': 'error_text', 'text': msg}) return (output_objects, returnvalues.CLIENT_ERROR) elif msg: # In case of warnings, msg is non-empty while ret_val remains True output_objects.append({'object_type': 'warning', 'text': msg}) # if we get here user is either vgrid owner or allowed to add rule # don't add if already in vgrid or parent vgrid - but update if owner update_id = None if vgrid_is_trigger(vgrid_name, rule_id, configuration): if vgrid_is_trigger_owner(vgrid_name, rule_id, client_id, configuration): update_id = 'rule_id' else: output_objects.append({ 'object_type': 'error_text', 'text': '%s is already a trigger owned by somebody else in the %s' % (rule_id, label) }) return (output_objects, returnvalues.CLIENT_ERROR) # don't add if already in subvgrid (list_status, subvgrids) = vgrid_list_subvgrids(vgrid_name, configuration) if not list_status: output_objects.append({ 'object_type': 'error_text', 'text': 'Error getting list of sub%ss: %s' % (label, subvgrids) }) return (output_objects, returnvalues.SYSTEM_ERROR) for subvgrid in subvgrids: if vgrid_is_trigger(subvgrid, rule_id, configuration, recursive=False): output_objects.append({ 'object_type': 'error_text', 'text': '''%(rule_id)s is already in a sub-%(vgrid_label)s (%(subvgrid)s). Please remove the trigger from the sub-%(vgrid_label)s and try again''' % { 'rule_id': rule_id, 'subvgrid': subvgrid, 'vgrid_label': label } }) return (output_objects, returnvalues.CLIENT_ERROR) if not action in valid_trigger_actions: output_objects.append({ 'object_type': 'error_text', 'text': "invalid action value %s" % action }) return (output_objects, returnvalues.CLIENT_ERROR) if keyword_all in changes: changes = valid_trigger_changes for change in changes: if not change in valid_trigger_changes: output_objects.append({ 'object_type': 'error_text', 'text': "found invalid change value %s" % change }) return (output_objects, returnvalues.CLIENT_ERROR) # Check if we should load saved trigger for rank change or update rule_dict = None if rank is not None or update_id is not None: (load_status, all_triggers) = vgrid_triggers(vgrid_name, configuration) if not load_status: output_objects.append({ 'object_type': 'error_text', 'text': 'Failed to load triggers for %s: %s' % (vgrid_name, all_triggers) }) return (output_objects, returnvalues.SYSTEM_ERROR) for saved_dict in all_triggers: if saved_dict['rule_id'] == rule_id: rule_dict = saved_dict break if rule_dict is None: output_objects.append({ 'object_type': 'error_text', 'text': 'No such trigger %s for %s: %s' % (rule_id, vgrid_name, all_triggers) }) return (output_objects, returnvalues.CLIENT_ERROR) elif not path: # New trigger with missing path output_objects.append({ 'object_type': 'error_text', 'text': '''Either path or rank must be set.''' }) return (output_objects, returnvalues.CLIENT_ERROR) elif action == "submit" and not arguments: # New submit trigger with missing mrsl arguments output_objects.append({ 'object_type': 'error_text', 'text': '''Submit triggers must give a job description file path as argument.''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Handle create and update (i.e. new, update all or just refresh mRSL) if rank is None: # IMPORTANT: we save the job template contents to avoid potential abuse # Otherwise someone else in the VGrid could tamper with the template # and make the next trigger execute arbitrary code on behalf of the # rule owner. templates = [] # Merge current and saved values req_dict = { 'rule_id': rule_id, 'vgrid_name': vgrid_name, 'path': path, 'changes': changes, 'run_as': client_id, 'action': action, 'arguments': arguments, 'rate_limit': rate_limit, 'settle_time': settle_time, 'match_files': match_files, 'match_dirs': match_dirs, 'match_recursive': match_recursive, 'templates': templates } if rule_dict is None: rule_dict = req_dict else: for field in user_arguments_dict: if field in req_dict: rule_dict[field] = req_dict[field] # Now refresh template contents if rule_dict['action'] == "submit": for rel_path in rule_dict['arguments']: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(os.path.join(base_dir, rel_path)) try: if not valid_user_path(configuration, abs_path, base_dir, True): logger.warning( '%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, rel_path)) raise ValueError('invalid submit path argument: %s' % rel_path) temp_fd = open(abs_path) templates.append(temp_fd.read()) temp_fd.close() except Exception as err: logger.error("read submit argument file failed: %s" % err) output_objects.append({ 'object_type': 'error_text', 'text': 'failed to read submit argument file "%s"' % rel_path }) return (output_objects, returnvalues.CLIENT_ERROR) # Save updated template contents here rule_dict['templates'] = templates # Add to list and pickle (add_status, add_msg) = vgrid_add_triggers(configuration, vgrid_name, [rule_dict], update_id, rank) if not add_status: logger.error('%s failed to add/update trigger: %s' % (client_id, add_msg)) output_objects.append({ 'object_type': 'error_text', 'text': '%s' % add_msg }) return (output_objects, returnvalues.SYSTEM_ERROR) if rank is not None: logger.info('%s moved trigger %s to %d' % (client_id, rule_id, rank)) output_objects.append({ 'object_type': 'text', 'text': 'moved %s trigger %s to position %d' % (vgrid_name, rule_id, rank) }) elif update_id: logger.info('%s updated trigger: %s' % (client_id, rule_dict)) output_objects.append({ 'object_type': 'text', 'text': 'Existing trigger %s successfully updated in %s %s!' % (rule_id, vgrid_name, label) }) else: logger.info('%s added new trigger: %s' % (client_id, rule_dict)) output_objects.append({ 'object_type': 'text', 'text': 'New trigger %s successfully added to %s %s!' % (rule_id, vgrid_name, label) }) output_objects.append({ 'object_type': 'link', 'destination': 'vgridworkflows.py?vgrid_name=%s' % vgrid_name, 'text': 'Back to workflows for %s' % vgrid_name }) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id) client_dir = client_id_dir(client_id) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, # NOTE: path can use wildcards, dst and current_dir cannot typecheck_overrides={'path': valid_path_pattern}, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) flags = ''.join(accepted['flags']) pattern_list = accepted['path'] dst = accepted['dst'][-1] current_dir = accepted['current_dir'][-1].lstrip(os.sep) # All paths are relative to current_dir pattern_list = [os.path.join(current_dir, i) for i in pattern_list] if dst: dst = os.path.join(current_dir, dst) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep status = returnvalues.OK if verbose(flags): for flag in flags: output_objects.append({ 'object_type': 'text', 'text': '%s using flag: %s' % (op_name, flag) }) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dir = os.path.abspath( os.path.join(base_dir, current_dir.lstrip(os.sep))) if not valid_user_path(configuration, abs_dir, base_dir, True): output_objects.append({ 'object_type': 'error_text', 'text': "You're not allowed to work in %s!" % current_dir }) logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_dir, current_dir)) return (output_objects, returnvalues.CLIENT_ERROR) if verbose(flags): output_objects.append({ 'object_type': 'text', 'text': "working in %s" % current_dir }) if dst: if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # NOTE: dst already incorporates current_dir prefix here # IMPORTANT: path must be expanded to abs for proper chrooting abs_dest = os.path.abspath(os.path.join(base_dir, dst)) logger.info('chksum in %s' % abs_dest) # Don't use abs_path in output as it may expose underlying # fs layout. relative_dest = abs_dest.replace(base_dir, '') if not valid_user_path(configuration, abs_dest, base_dir, True): output_objects.append({ 'object_type': 'error_text', 'text': "Invalid path! (%s expands to an illegal path)" % dst }) logger.warning('%s tried to %s restricted path %s !(%s)' % (client_id, op_name, abs_dest, dst)) return (output_objects, returnvalues.CLIENT_ERROR) if not check_write_access(abs_dest, parent_dir=True): logger.warning('%s called without write access: %s' % (op_name, abs_dest)) output_objects.append({ 'object_type': 'error_text', 'text': 'cannot checksum to "%s": inside a read-only location!' % relative_dest }) return (output_objects, returnvalues.CLIENT_ERROR) all_lines = [] for pattern in pattern_list: # Check directory traversal attempts before actual handling to avoid # leaking information about file system layout while allowing # consistent error messages unfiltered_match = glob.glob(base_dir + pattern) match = [] for server_path in unfiltered_match: # IMPORTANT: path must be expanded to abs for proper chrooting abs_path = os.path.abspath(server_path) if not valid_user_path(configuration, abs_path, base_dir, True): # out of bounds - save user warning for later to allow # partial match: # ../*/* is technically allowed to match own files. logger.warning('%s tried to %s restricted path %s ! (%s)' % (client_id, op_name, abs_path, pattern)) continue match.append(abs_path) # Now actually treat list of allowed matchings and notify if no # (allowed) match if not match: output_objects.append({ 'object_type': 'file_not_found', 'name': pattern }) status = returnvalues.FILE_NOT_FOUND # NOTE: we produce output matching an invocation of: # du -aL --apparent-size --block-size=1 PATH [PATH ...] filedus = [] summarize_output = summarize(flags) for abs_path in match: if invisible_path(abs_path): continue relative_path = abs_path.replace(base_dir, '') # cache accumulated sub dir sizes - du sums into parent dir size dir_sizes = {} try: # Assume a directory to walk for (root, dirs, files) in walk(abs_path, topdown=False, followlinks=True): if invisible_path(root): continue dir_bytes = 0 for name in files: real_file = os.path.join(root, name) if invisible_path(real_file): continue relative_file = real_file.replace(base_dir, '') size = os.path.getsize(real_file) dir_bytes += size if not summarize_output: filedus.append({ 'object_type': 'filedu', 'name': relative_file, 'bytes': size }) for name in dirs: real_dir = os.path.join(root, name) if invisible_path(real_dir): continue dir_bytes += dir_sizes[real_dir] relative_root = root.replace(base_dir, '') dir_bytes += os.path.getsize(root) dir_sizes[root] = dir_bytes if root == abs_path or not summarize_output: filedus.append({ 'object_type': 'filedu', 'name': relative_root, 'bytes': dir_bytes }) if os.path.isfile(abs_path): # Fall back to plain file where walk is empty size = os.path.getsize(abs_path) filedus.append({ 'object_type': 'filedu', 'name': relative_path, 'bytes': size }) except Exception as exc: output_objects.append({ 'object_type': 'error_text', 'text': "%s: '%s': %s" % (op_name, relative_path, exc) }) logger.error("%s: failed on '%s': %s" % (op_name, relative_path, exc)) status = returnvalues.SYSTEM_ERROR continue if dst: all_lines += [ '%(bytes)d\t\t%(name)s\n' % entry for entry in filedus ] else: output_objects.append({ 'object_type': 'filedus', 'filedus': filedus }) if dst and not write_file(''.join(all_lines), abs_dest, logger): output_objects.append({ 'object_type': 'error_text', 'text': "failed to write disk use to %s" % relative_dest }) logger.error("writing disk use to %s for %s failed" % (abs_dest, client_id)) status = returnvalues.SYSTEM_ERROR return (output_objects, status)
def run_transfer(configuration, client_id, transfer_dict): """Actual data transfer built from transfer_dict on behalf of client_id""" logger.debug('run transfer for %s: %s' % (client_id, blind_pw(transfer_dict))) transfer_id = transfer_dict['transfer_id'] action = transfer_dict['action'] protocol = transfer_dict['protocol'] status_dir = get_status_dir(configuration, client_id, transfer_id) cmd_map = get_cmd_map() if not protocol in cmd_map[action]: raise ValueError('unsupported protocol: %s' % protocol) client_dir = client_id_dir(client_id) makedirs_rec(status_dir, configuration) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep # TODO: we should refactor to move command extraction into one function command_pattern = cmd_map[action][protocol] target_helper_list = [] key_path = transfer_dict.get("key", "") if key_path: # Use key with given name from settings dir settings_base_dir = os.path.abspath( os.path.join(configuration.user_settings, client_dir)) + os.sep key_path = os.path.join(settings_base_dir, user_keys_dir, key_path.lstrip(os.sep)) # IMPORTANT: path must be expanded to abs for proper chrooting key_path = os.path.abspath(key_path) if not valid_user_path(configuration, key_path, settings_base_dir): logger.error('rejecting illegal directory traversal for %s (%s)' % (key_path, blind_pw(transfer_dict))) raise ValueError("user provided a key outside own settings!") rel_src_list = transfer_dict['src'] rel_dst = transfer_dict['dst'] compress = transfer_dict.get("compress", False) exclude = transfer_dict.get("exclude", []) if transfer_dict['action'] in ('import', ): logger.debug('setting abs dst for action %(action)s' % transfer_dict) src_path_list = transfer_dict['src'] dst_path = os.path.join(base_dir, rel_dst.lstrip(os.sep)) dst_path = os.path.abspath(dst_path) for src in rel_src_list: abs_dst = os.path.join(dst_path, src.lstrip(os.sep)) # IMPORTANT: path must be expanded to abs for proper chrooting abs_dst = os.path.abspath(abs_dst) # Reject illegal directory traversal and hidden files if not valid_user_path(configuration, abs_dst, base_dir, True): logger.error( 'rejecting illegal directory traversal for %s (%s)' % (abs_dst, blind_pw(transfer_dict))) raise ValueError("user provided a destination outside home!") if src.endswith(os.sep): target_helper_list.append( (get_lftp_target(True, False, exclude), get_rsync_target(True, False, exclude, compress))) else: target_helper_list.append( (get_lftp_target(True, True, exclude), get_rsync_target(True, True, exclude, compress))) makedirs_rec(dst_path, configuration) elif transfer_dict['action'] in ('export', ): logger.debug('setting abs src for action %(action)s' % transfer_dict) dst_path = transfer_dict['dst'] src_path_list = [] for src in rel_src_list: src_path = os.path.join(base_dir, src.lstrip(os.sep)) # IMPORTANT: path must be expanded to abs for proper chrooting src_path = os.path.abspath(src_path) # Reject illegal directory traversal and hidden files if not valid_user_path(configuration, src_path, base_dir, True): logger.error( 'rejecting illegal directory traversal for %s (%s)' % (src, blind_pw(transfer_dict))) raise ValueError("user provided a source outside home!") src_path_list.append(src_path) if src.endswith(os.sep) or os.path.isdir(src): target_helper_list.append( (get_lftp_target(False, False, exclude), get_rsync_target(False, False, exclude, compress))) else: target_helper_list.append( (get_lftp_target(False, True, exclude), get_rsync_target(False, True, exclude, compress))) else: raise ValueError('unsupported action for %(transfer_id)s: %(action)s' % transfer_dict) run_dict = transfer_dict.copy() run_dict['log_path'] = os.path.join(status_dir, 'transfer.log') # Use private known hosts file for ssh transfers as explained above # NOTE: known_hosts containing '=' silently leads to rest getting ignored! # use /dev/null to skip host key verification completely for now. #run_dict['known_hosts'] = os.path.join(base_dir, '.ssh', 'known_hosts') run_dict['known_hosts'] = '/dev/null' # Make sure password is set to empty string as default run_dict['password'] = run_dict.get('password', '') # TODO: this is a bogus cert path for now - we don't support ssl certs run_dict['cert'] = run_dict.get('cert', '') # IMPORTANT: must be implicit proto or 'ftp://' (not ftps://) and similarly # webdav(s) must use explicit http(s) instead. In both cases we # replace protocol between cmd selection and lftp path expansion if run_dict['protocol'] == 'ftps': run_dict['orig_proto'] = run_dict['protocol'] run_dict['protocol'] = 'ftp' logger.info( 'force %(orig_proto)s to %(protocol)s for %(transfer_id)s' % run_dict) elif run_dict['protocol'].startswith('webdav'): run_dict['orig_proto'] = run_dict['protocol'] run_dict['protocol'] = run_dict['protocol'].replace('webdav', 'http') logger.info( 'force %(orig_proto)s to %(protocol)s for %(transfer_id)s' % run_dict) if key_path: rel_key = run_dict['key'] rel_cert = run_dict['cert'] run_dict['key'] = key_path run_dict['cert'] = key_path.replace(rel_key, rel_cert) run_dict['ssh_auth'] = get_ssh_auth(True, run_dict) run_dict['ssl_auth'] = get_ssl_auth(True, run_dict) else: # Extract encrypted password password_digest = run_dict.get('password_digest', '') if password_digest: _, _, _, payload = password_digest.split("$") unscrambled = unscramble_digest(configuration.site_digest_salt, payload) _, _, password = unscrambled.split(":") run_dict['password'] = password run_dict['ssh_auth'] = get_ssh_auth(False, run_dict) run_dict['ssl_auth'] = get_ssl_auth(False, run_dict) run_dict['rel_dst'] = rel_dst run_dict['dst'] = dst_path run_dict['lftp_buf_size'] = run_dict.get('lftp_buf_size', lftp_buffer_bytes) run_dict['lftp_sftp_block_size'] = run_dict.get('sftp_sftp_block_size', lftp_sftp_block_bytes) status = 0 for (src, rel_src, target_helper) in zip(src_path_list, rel_src_list, target_helper_list): (lftp_target, rsync_target) = target_helper logger.debug('setting up %(action)s for %(src)s' % run_dict) if run_dict['protocol'] == 'sftp' and not os.path.isabs(src): # NOTE: lftp interprets sftp://FQDN/SRC as absolute path /SRC # We force relative paths into user home with a tilde. # The resulting sftp://FQDN/~/SRC looks funky but works. run_dict['src'] = "~/%s" % src else: # All other paths are probably absolute or auto-chrooted anyway run_dict['src'] = src run_dict['rel_src'] = rel_src run_dict['lftp_args'] = ' '.join(lftp_target[0]) % run_dict run_dict['lftp_excludes'] = ' '.join(lftp_target[1]) # src and dst may actually be reversed for lftp, but for symmetry ... run_dict['lftp_src'] = lftp_target[2][0] % run_dict run_dict['lftp_dst'] = lftp_target[2][1] % run_dict run_dict['rsync_args'] = ' '.join(rsync_target[0]) % run_dict # Preserve excludes on list form for rsync, where it matters run_dict[RSYNC_EXCLUDES_LIST] = rsync_target[1] run_dict['rsync_src'] = rsync_target[2][0] % run_dict run_dict['rsync_dst'] = rsync_target[2][1] % run_dict blind_dict = blind_pw(run_dict) logger.debug('expanded vars to %s' % blind_dict) # NOTE: Make sure NOT to break rsync excludes on list form as they # won't work if concatenated to a single string in command_list! command_list, blind_list = [], [] for i in command_pattern: if i == RSYNC_EXCLUDES_LIST: command_list += run_dict[RSYNC_EXCLUDES_LIST] blind_list += run_dict[RSYNC_EXCLUDES_LIST] else: command_list.append(i % run_dict) blind_list.append(i % blind_dict) command_str = ' '.join(command_list) blind_str = ' '.join(blind_list) logger.info('run %s on behalf of %s' % (blind_str, client_id)) transfer_proc = subprocess_popen(command_list, stdout=subprocess_pipe, stderr=subprocess_pipe) # Save transfer_proc.pid for use in clean up during shutdown # in that way we can resume pretty smoothly in next run. sub_pid = transfer_proc.pid logger.info('%s %s running transfer process %s' % (client_id, transfer_id, sub_pid)) add_sub_pid(configuration, sub_pid_map, client_id, transfer_id, sub_pid) out, err = transfer_proc.communicate() exit_code = transfer_proc.wait() status |= exit_code del_sub_pid(configuration, sub_pid_map, client_id, transfer_id, sub_pid) logger.info('done running transfer %s: %s' % (transfer_id, blind_str)) logger.debug('raw output is: %s' % out) logger.debug('raw error is: %s' % err) logger.debug('result was %s' % exit_code) if not transfer_result(configuration, client_id, run_dict, exit_code, out.replace(base_dir, ''), err.replace(base_dir, '')): logger.error('writing transfer status for %s failed' % transfer_id) logger.debug('done handling transfers in %(transfer_id)s' % transfer_dict) transfer_dict['exit_code'] = status if status == 0: transfer_dict['status'] = 'DONE' else: transfer_dict['status'] = 'FAILED'