def is_owner(
client_id,
unique_config_name,
config_home,
logger,
):
"""Check that client_id is listed in pickled owners file"""
config_path = os.path.abspath(os.path.join(config_home,
unique_config_name, 'owners'))
# Check validity of unique_config_name
# Automatic configuration extraction
configuration = None
if not valid_user_path(configuration, config_path, config_home):
# Extract caller information
from traceback import format_stack
caller = ''.join(format_stack()[:-1]).strip()
logger.warning("""is_owner caught possible illegal directory traversal attempt
by client: '%s'
unique name: '%s'
caller: %s"""
% (client_id, unique_config_name, caller))
return False
return is_item_in_pickled_list(config_path, client_id, logger)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = fileinfo_dict['base_path'] = fileinfo_dict['base_path'] + os.sep
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
path = fileinfo_dict['path']
unfiltered_match = [base_dir + os.sep + fileinfo_dict['path']]
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to access restricted path %s ! (%s)' %
(client_id, abs_path, path))
o.out('Access to %s is prohibited!' % fileinfo_dict['path'])
o.reply_and_exit(o.CLIENT_ERROR)
if action == 'GET':
status = get(o, fileinfo_dict)
# if status, we have already written to the client, see get method.
if not status:
o.reply_and_exit(o.ERROR)
elif action == 'PUT':
status = put(o, fileinfo_dict)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path cannot use wildcards here
typecheck_overrides={},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
path = accepted['path'][-1]
chosen_newline = accepted['newline'][-1]
submitjob = accepted['submitjob'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs and submitjob:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
# HTML spec dictates newlines in forms to be MS style (\r\n)
# rather than un*x style (\n): change if requested.
form_newline = '\r\n'
allowed_newline = {'unix': '\n', 'mac': '\r', 'windows': '\r\n'}
output_objects.append({
'object_type': 'header',
'text': 'Saving changes to edited file'
})
if not chosen_newline in allowed_newline.keys():
output_objects.append({
'object_type':
'error_text',
'text':
'Unsupported newline style supplied: %s (must be one of %s)' %
(chosen_newline, ', '.join(allowed_newline.keys()))
})
return (output_objects, returnvalues.CLIENT_ERROR)
saved_newline = allowed_newline[chosen_newline]
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing consistent
# error messages
abs_path = ''
unfiltered_match = glob.glob(base_dir + path)
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, path))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
if abs_path == '':
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, path.lstrip(os.sep)))
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, path))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_path, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_path))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot edit "%s": inside a read-only location!' % path
})
status = returnvalues.CLIENT_ERROR
return (output_objects, returnvalues.CLIENT_ERROR)
(owner, time_left) = acquire_edit_lock(abs_path, client_id)
if owner != client_id:
output_objects.append({
'object_type': 'error_text',
'text': "You don't have the lock for %s!" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
try:
fh = open(abs_path, 'w+')
fh.write(user_arguments_dict['editarea'][0].replace(
form_newline, saved_newline))
fh.close()
# everything ok
output_objects.append({
'object_type': 'text',
'text': 'Saved changes to %s.' % path
})
logger.info('saved changes to %s' % path)
release_edit_lock(abs_path, client_id)
except Exception as exc:
# Don't give away information about actual fs layout
output_objects.append({
'object_type':
'error_text',
'text':
'%s could not be written! (%s)' %
(path, str(exc).replace(base_dir, ''))
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if submitjob:
output_objects.append({
'object_type': 'text',
'text': 'Submitting saved file to parser'
})
submitstatus = {'object_type': 'submitstatus', 'name': path}
(new_job_status, msg, job_id) = new_job(abs_path, client_id,
configuration, False, True)
if not new_job_status:
submitstatus['status'] = False
submitstatus['message'] = msg
else:
submitstatus['status'] = True
submitstatus['job_id'] = job_id
output_objects.append({
'object_type': 'submitstatuslist',
'submitstatuslist': [submitstatus]
})
output_objects.append({
'object_type': 'link',
'destination': 'javascript:history.back()',
'class': 'backlink iconspace',
'title': 'Go back to previous page',
'text': 'Back to previous page'
})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path can use wildcards
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
search = accepted['pattern'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text':
'%s using flag: %s' % (op_name, flag)})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
output_lines = []
try:
matching = pattern_match_file(search, abs_path)
for line in matching:
output_lines.append(line)
except Exception as exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
return (output_objects, status)
# Single file sharelinks use direct link to file. If so we
# manually expand to direct target. Otherwise we only replace
# that prefix of path to translate it to a sharelink dir path.
if is_file:
logger.debug("found single file link: %s" % path)
path = link_target
else:
logger.debug("found directory link: %s" % path)
path = path.replace(full_prefix, link_target, 1)
real_path = os.path.realpath(path)
logger.info("check path from %s in base %s or chroot: %s" %
(client_ip, base_path, path))
# Exact match to sid dir does not make sense as we expect a file
# IMPORTANT: use path and not real_path here in order to test both
if not valid_user_path(configuration, path, base_path,
allow_equal=is_file, apache_scripts=True):
logger.error("request from %s is outside sid chroot %s: %s (%s)" %
(client_ip, base_path, raw_path, real_path))
print(INVALID_MARKER)
continue
logger.info("found valid sid chroot path from %s: %s" %
(client_ip, real_path))
print(real_path)
# Throttle down a bit to yield
time.sleep(0.01)
except KeyboardInterrupt:
keep_running = False
except Exception as exc:
logger.error("unexpected exception: %s" % exc)
def main(client_id, user_arguments_dict, environ=None):
"""Main function used by front end"""
if environ is None:
environ = os.environ
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False,
op_menu=client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input(
user_arguments_dict,
defaults,
output_objects,
allow_rejects=False,
# NOTE: path cannot use wildcards here
typecheck_overrides={},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
current_dir = accepted['current_dir'][-1]
share_id = accepted['share_id'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Either authenticated user client_id set or sharelink ID
if client_id:
user_id = client_id
target_dir = client_id_dir(client_id)
base_dir = configuration.user_home
id_query = ''
page_title = 'Create User Directory'
userstyle = True
widgets = True
elif share_id:
try:
(share_mode, _) = extract_mode_id(configuration, share_id)
except ValueError as err:
logger.error('%s called with invalid share_id %s: %s' %
(op_name, share_id, err))
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid sharelink ID: %s' % share_id
})
return (output_objects, returnvalues.CLIENT_ERROR)
# TODO: load and check sharelink pickle (currently requires client_id)
user_id = 'anonymous user through share ID %s' % share_id
if share_mode == 'read-only':
logger.error('%s called without write access: %s' %
(op_name, accepted))
output_objects.append({
'object_type': 'error_text',
'text': 'No write access!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
target_dir = os.path.join(share_mode, share_id)
base_dir = configuration.sharelink_home
id_query = '?share_id=%s' % share_id
page_title = 'Create Shared Directory'
userstyle = False
widgets = False
else:
logger.error('%s called without proper auth: %s' % (op_name, accepted))
output_objects.append({
'object_type': 'error_text',
'text': 'Authentication is missing!'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(base_dir, target_dir)) + os.sep
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = page_title
title_entry['skipwidgets'] = not widgets
title_entry['skipuserstyle'] = not userstyle
output_objects.append({'object_type': 'header', 'text': page_title})
# Input validation assures target_dir can't escape base_dir
if not os.path.isdir(base_dir):
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid client/sharelink id!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
# NB: Globbing disabled on purpose here
unfiltered_match = [base_dir + os.sep + current_dir + os.sep + pattern]
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warn('%s tried to %s %s restricted path! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: cannot create directory '%s': Permission denied" %
(op_name, pattern)
})
status = returnvalues.CLIENT_ERROR
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({
'object_type': 'file',
'name': relative_path
})
if not parents(flags) and os.path.exists(abs_path):
output_objects.append({
'object_type': 'error_text',
'text': '%s: path exist!' % pattern
})
status = returnvalues.CLIENT_ERROR
continue
if not check_write_access(abs_path, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_path))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot create "%s": inside a read-only location!' %
pattern
})
status = returnvalues.CLIENT_ERROR
continue
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
'created', [relative_path])
if parents(flags):
if not os.path.isdir(abs_path):
os.makedirs(abs_path)
else:
os.mkdir(abs_path)
logger.info('%s %s done' % (op_name, abs_path))
except Exception as exc:
if not isinstance(exc, GDPIOLogError):
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'created', [relative_path],
failed=True,
details=exc)
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s' failed!" % (op_name, relative_path)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({
'object_type':
'text',
'text':
"created directory %s" % (relative_path)
})
if id_query:
open_query = "%s;current_dir=%s" % (id_query, relative_path)
else:
open_query = "?current_dir=%s" % relative_path
output_objects.append({
'object_type': 'link',
'destination': 'ls.py%s' % open_query,
'text': 'Open %s' % relative_path
})
output_objects.append({'object_type': 'text', 'text': ''})
output_objects.append({
'object_type': 'link',
'destination': 'ls.py%s' % id_query,
'text': 'Return to files overview'
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
patterns = accepted['job_id']
action = accepted['action'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not action in valid_actions.keys():
output_objects.append({
'object_type':
'error_text',
'text':
'Invalid job action "%s" (only %s supported)' %
(action, ', '.join(valid_actions.keys()))
})
return (output_objects, returnvalues.CLIENT_ERROR)
new_state = valid_actions[action]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
status = returnvalues.OK
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.error(
'%s tried to use %s %s outside own home! (pattern %s)' %
(client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % pattern
})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
# job state change is hard on the server, limit
if len(filelist) > 500:
output_objects.append({
'object_type':
'error_text',
'text':
'Too many matching jobs (%s)!' % len(filelist)
})
return (output_objects, returnvalues.CLIENT_ERROR)
changedstatusjobs = []
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
changedstatusjob = {
'object_type': 'changedstatusjob',
'job_id': job_id
}
job_dict = unpickle(filepath, logger)
if not job_dict:
changedstatusjob['message'] = '''The file containing the
information for job id %s could not be opened! You can only %s your own
jobs!''' % (job_id, action)
changedstatusjobs.append(changedstatusjob)
status = returnvalues.CLIENT_ERROR
continue
changedstatusjob['oldstatus'] = job_dict['STATUS']
# Is the job status compatible with action?
possible_cancel_states = [
'PARSE', 'QUEUED', 'RETRY', 'EXECUTING', 'FROZEN'
]
if action == 'cancel' and \
not job_dict['STATUS'] in possible_cancel_states:
changedstatusjob['message'] = \
'You can only cancel jobs with status: %s.'\
% ' or '.join(possible_cancel_states)
status = returnvalues.CLIENT_ERROR
changedstatusjobs.append(changedstatusjob)
continue
possible_freeze_states = ['QUEUED', 'RETRY']
if action == 'freeze' and \
not job_dict['STATUS'] in possible_freeze_states:
changedstatusjob['message'] = \
'You can only freeze jobs with status: %s.'\
% ' or '.join(possible_freeze_states)
status = returnvalues.CLIENT_ERROR
changedstatusjobs.append(changedstatusjob)
continue
possible_thaw_states = ['FROZEN']
if action == 'thaw' and \
not job_dict['STATUS'] in possible_thaw_states:
changedstatusjob['message'] = \
'You can only thaw jobs with status: %s.'\
% ' or '.join(possible_thaw_states)
status = returnvalues.CLIENT_ERROR
changedstatusjobs.append(changedstatusjob)
continue
# job action is handled by changing the STATUS field, notifying the
# job queue and making sure the server never submits jobs with status
# FROZEN or CANCELED.
# file is repickled to ensure newest information is used, job_dict
# might be old if another script has modified the file.
if not unpickle_and_change_status(filepath, new_state, logger):
output_objects.append({
'object_type':
'error_text',
'text':
'Job status could not be changed to %s!' % new_state
})
status = returnvalues.SYSTEM_ERROR
# Avoid key error and make sure grid_script gets expected number of
# arguments
if 'UNIQUE_RESOURCE_NAME' not in job_dict:
job_dict['UNIQUE_RESOURCE_NAME'] = \
'UNIQUE_RESOURCE_NAME_NOT_FOUND'
if 'EXE' not in job_dict:
job_dict['EXE'] = 'EXE_NAME_NOT_FOUND'
# notify queue
if not send_message_to_grid_script(
'JOBACTION ' + job_id + ' ' + job_dict['STATUS'] + ' ' +
new_state + ' ' + job_dict['UNIQUE_RESOURCE_NAME'] + ' ' +
job_dict['EXE'] + '\n', logger, configuration):
output_objects.append({
'object_type':
'error_text',
'text':
'''Error sending message to grid_script,
job may still be in the job queue.'''
})
status = returnvalues.SYSTEM_ERROR
continue
changedstatusjob['newstatus'] = new_state
changedstatusjobs.append(changedstatusjob)
output_objects.append({
'object_type': 'changedstatusjobs',
'changedstatusjobs': changedstatusjobs
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
defaults = signature()[1]
output_objects.append({
'object_type': 'header',
'text': 'Reject Resource Request'
})
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
unique_resource_name = accepted['unique_resource_name'][-1].strip()
request_name = unhexlify(accepted['request_name'][-1])
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not is_owner(client_id, unique_resource_name,
configuration.resource_home, logger):
output_objects.append({
'object_type':
'error_text',
'text':
'You must be an owner of %s to reject requests!' %
unique_resource_name
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# resource dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.resource_home,
unique_resource_name)) + os.sep
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, request_name))
if not valid_user_path(
configuration, abs_path, base_dir, allow_equal=False):
logger.warning('%s tried to access restricted path %s ! (%s)' % \
(client_id, abs_path, request_name))
output_objects.append({
'object_type':
'error_text',
'text':
'''Illegal request name "%s":
you can only reject requests to your own resources.''' % request_name
})
return (output_objects, returnvalues.CLIENT_ERROR)
if request_name:
request_dir = os.path.join(configuration.resource_home,
unique_resource_name)
req = load_access_request(configuration, request_dir, request_name)
if not req or not delete_access_request(configuration, request_dir,
request_name):
logger.error("failed to delete owner request for %s in %s" % \
(unique_resource_name, request_name))
output_objects.append({
'object_type': 'error_text', 'text':
'Failed to remove saved resource request for %s in %s!'\
% (unique_resource_name, request_name)})
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append({
'object_type':
'text',
'text':
'''
Deleted %(request_type)s access request to %(target)s for %(entity)s .
''' % req
})
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
fill_helpers = {
'protocol': any_protocol,
'unique_resource_name': unique_resource_name,
'form_method': form_method,
'csrf_field': csrf_field,
'csrf_limit': csrf_limit
}
fill_helpers.update(req)
target_op = 'sendrequestaction'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token})
output_objects.append({
'object_type':
'html_form',
'text':
"""
<p>
You can use the reply form below if you want to additionally send an
explanation for rejecting the request.
</p>
<form method='%(form_method)s' action='%(target_op)s.py'>
<input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' />
<input type=hidden name=request_type value='resourcereject' />
<input type=hidden name=unique_resource_name value='%(target)s' />
<input type=hidden name=cert_id value='%(entity)s' />
<input type=hidden name=protocol value='%(protocol)s' />
<table>
<tr>
<td class='title'>Optional reject message to requestor(s)</td>
</tr><tr>
<td><textarea name=request_text cols=72 rows=10>
We have decided to reject your %(request_type)s request to our %(target)s
resource.
Regards, the %(target)s resource owners
</textarea></td>
</tr>
<tr>
<td><input type='submit' value='Inform requestor(s)' /></td>
</tr>
</table>
</form>
<br />
""" % fill_helpers
})
output_objects.append({
'object_type':
'link',
'destination':
'resadmin.py?unique_resource_name=%s' % unique_resource_name,
'text':
'Back to administration for %s' % unique_resource_name
})
return (output_objects, returnvalues.OK)
# outside home, which is checked later.
path = os.path.abspath(path)
if not path.startswith(home_path):
logger.error("got path from %s outside user home: %s" %
(client_ip, raw_path))
print(INVALID_MARKER)
continue
real_path = os.path.realpath(path)
logger.debug("check path %s in home %s or chroot" %
(path, home_path))
# Exact match to user home does not make sense as we expect a file
# IMPORTANT: use path and not real_path here in order to test both
if not valid_user_path(configuration,
path,
home_path,
allow_equal=False,
apache_scripts=True):
logger.error("path from %s outside user chroot %s: %s (%s)" %
(client_ip, home_path, raw_path, real_path))
print(INVALID_MARKER)
continue
elif not check_account_accessible(configuration, user_id, 'https'):
# Only warn to avoid excessive noise from scanners
logger.warning(
"path from %s in inaccessible %s account: %s (%s)" %
(client_ip, user_id, raw_path, real_path))
print(INVALID_MARKER)
continue
logger.info("found valid user chroot path from %s: %s" %
def main(client_id, user_arguments_dict, environ=None):
"""Main function used by front end"""
if environ is None:
environ = os.environ
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: src and dst can use wildcards here
typecheck_overrides={'src': valid_path_pattern,
'dst': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
src_list = accepted['src']
dst = accepted['dst'][-1]
iosessionid = accepted['iosessionid'][-1]
share_id = accepted['share_id'][-1]
freeze_id = accepted['freeze_id'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append(
{'object_type': 'error_text', 'text': '''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
# Special handling if used from a job (no client_id but iosessionid)
if not client_id and iosessionid:
base_dir = os.path.realpath(configuration.webserver_home
+ os.sep + iosessionid) + os.sep
# Use selected base as source and destination dir by default
src_base = dst_base = base_dir
# Sharelink import if share_id is given - change to sharelink as src base
if share_id:
try:
(share_mode, _) = extract_mode_id(configuration, share_id)
except ValueError as err:
logger.error('%s called with invalid share_id %s: %s' %
(op_name, share_id, err))
output_objects.append(
{'object_type': 'error_text', 'text':
'Invalid sharelink ID: %s' % share_id})
return (output_objects, returnvalues.CLIENT_ERROR)
# TODO: load and check sharelink pickle (currently requires client_id)
if share_mode == 'write-only':
logger.error('%s called import from write-only sharelink: %s'
% (op_name, accepted))
output_objects.append(
{'object_type': 'error_text', 'text':
'Sharelink %s is write-only!' % share_id})
return (output_objects, returnvalues.CLIENT_ERROR)
target_dir = os.path.join(share_mode, share_id)
src_base = os.path.abspath(os.path.join(configuration.sharelink_home,
target_dir)) + os.sep
if os.path.isfile(os.path.realpath(src_base)):
logger.error('%s called import on single file sharelink: %s'
% (op_name, share_id))
output_objects.append(
{'object_type': 'error_text', 'text': """Import is only
supported for directory sharelinks!"""})
return (output_objects, returnvalues.CLIENT_ERROR)
elif not os.path.isdir(src_base):
logger.error('%s called import with non-existant sharelink: %s'
% (client_id, share_id))
output_objects.append(
{'object_type': 'error_text', 'text': 'No such sharelink: %s'
% share_id})
return (output_objects, returnvalues.CLIENT_ERROR)
# Archive import if freeze_id is given - change to archive as src base
if freeze_id:
if not is_frozen_archive(client_id, freeze_id, configuration):
logger.error('%s called with invalid freeze_id: %s' %
(op_name, freeze_id))
output_objects.append(
{'object_type': 'error_text', 'text':
'Invalid archive ID: %s' % freeze_id})
return (output_objects, returnvalues.CLIENT_ERROR)
target_dir = os.path.join(client_dir, freeze_id)
src_base = os.path.abspath(os.path.join(configuration.freeze_home,
target_dir)) + os.sep
if not os.path.isdir(src_base):
logger.error('%s called import with non-existant archive: %s'
% (client_id, freeze_id))
output_objects.append(
{'object_type': 'error_text', 'text': 'No such archive: %s'
% freeze_id})
return (output_objects, returnvalues.CLIENT_ERROR)
status = returnvalues.OK
abs_dest = dst_base + dst
dst_list = glob.glob(abs_dest)
if not dst_list:
# New destination?
if not glob.glob(os.path.dirname(abs_dest)):
logger.error('%s called with illegal dst: %s'
% (op_name, dst))
output_objects.append(
{'object_type': 'error_text', 'text':
'Illegal dst path provided!'})
return (output_objects, returnvalues.CLIENT_ERROR)
else:
dst_list = [abs_dest]
# Use last match in case of multiple matches
dest = dst_list[-1]
if len(dst_list) > 1:
output_objects.append(
{'object_type': 'warning', 'text':
'dst (%s) matches multiple targets - using last: %s'
% (dst, dest)})
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(dst_base, '')
if not valid_user_path(configuration, abs_dest, dst_base, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_dest, dst))
output_objects.append(
{'object_type': 'error_text', 'text':
"Invalid destination (%s expands to an illegal path)" % dst})
return (output_objects, returnvalues.CLIENT_ERROR)
# We must make sure target dir exists if called in import X mode
if (share_id or freeze_id) and not makedirs_rec(abs_dest, configuration):
logger.error('could not create import destination dir: %s' % abs_dest)
output_objects.append(
{'object_type': 'error_text', 'text':
'cannot import to "%s" : file in the way?' % relative_dest})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s'
% (op_name, abs_dest))
output_objects.append(
{'object_type': 'error_text', 'text':
'cannot copy to "%s": inside a read-only location!'
% relative_dest})
return (output_objects, returnvalues.CLIENT_ERROR)
if share_id and not force(flags) and not check_empty_dir(abs_dest):
logger.warning('%s called %s sharelink import with non-empty dst: %s'
% (op_name, share_id, abs_dest))
output_objects.append(
{'object_type': 'error_text', 'text':
"""Importing a sharelink like '%s' into the non-empty '%s' folder
will potentially overwrite existing files with the sharelink version. If you
really want that, please try import again and select the overwrite box to
confirm it. You may want to back up any important data from %s first, however.
""" % (share_id, relative_dest, relative_dest)})
return (output_objects, returnvalues.CLIENT_ERROR)
if freeze_id and not force(flags) and not check_empty_dir(abs_dest):
logger.warning('%s called %s archive import with non-empty dst: %s'
% (op_name, freeze_id, abs_dest))
output_objects.append(
{'object_type': 'error_text', 'text':
"""Importing an archive like '%s' into the non-empty '%s' folder
will potentially overwrite existing files with the archive version. If you
really want that, please try import again and select the overwrite box to
confirm it. You may want to back up any important data from %s first, however.
""" % (freeze_id, relative_dest, relative_dest)})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in src_list:
unfiltered_match = glob.glob(src_base + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, src_base, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(src_base, '')
if verbose(flags):
output_objects.append(
{'object_type': 'file', 'name': relative_path})
# Prevent vgrid share copy which would create read-only dot dirs
# Generally refuse handling symlinks including root vgrid shares
if os.path.islink(abs_path):
output_objects.append(
{'object_type': 'warning', 'text': """You're not allowed to
copy entire special folders like %s shared folders!"""
% configuration.site_vgrid_label})
status = returnvalues.CLIENT_ERROR
continue
elif os.path.realpath(abs_path) == os.path.realpath(base_dir):
logger.error("%s: refusing copy home dir: %s" % (op_name,
abs_path))
output_objects.append(
{'object_type': 'warning', 'text':
"You're not allowed to copy your entire home directory!"
})
status = returnvalues.CLIENT_ERROR
continue
# src must be a file unless recursive is specified
if not recursive(flags) and os.path.isdir(abs_path):
logger.warning('skipping directory source %s' % abs_path)
output_objects.append(
{'object_type': 'warning', 'text':
'skipping directory src %s!' % relative_path})
continue
# If destination is a directory the src should be copied there
abs_target = abs_dest
if os.path.isdir(abs_target):
abs_target = os.path.join(abs_target,
os.path.basename(abs_path))
if os.path.abspath(abs_path) == os.path.abspath(abs_target):
logger.warning('%s tried to %s %s to itself! (%s)'
% (client_id, op_name, abs_path, pattern))
output_objects.append(
{'object_type': 'warning', 'text':
"Cannot copy '%s' to self!" % relative_path})
status = returnvalues.CLIENT_ERROR
continue
if os.path.isdir(abs_path) and \
abs_target.startswith(abs_path + os.sep):
logger.warning('%s tried to %s %s to itself! (%s)'
% (client_id, op_name, abs_path, pattern))
output_objects.append(
{'object_type': 'warning', 'text':
"Cannot copy '%s' to (sub) self!" % relative_path})
status = returnvalues.CLIENT_ERROR
continue
try:
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'copied',
[relative_path,
relative_dest
+ "/" + os.path.basename(relative_path)])
if os.path.isdir(abs_path):
shutil.copytree(abs_path, abs_target)
else:
shutil.copy(abs_path, abs_target)
logger.info('%s %s %s done' % (op_name, abs_path, abs_target))
except Exception as exc:
if not isinstance(exc, GDPIOLogError):
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'copied',
[relative_path,
relative_dest
+ "/" + os.path.basename(relative_path)],
failed=True,
details=exc)
output_objects.append(
{'object_type': 'error_text',
'text': "%s: failed on '%s' to '%s'"
% (op_name, relative_path, relative_dest)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
return (output_objects, status)
def main(client_id, user_arguments_dict, environ=None):
"""Main function used by front end"""
if environ is None:
environ = os.environ
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: src and dst can use wildcards here
typecheck_overrides={
'src': valid_path_pattern,
'dst': valid_path_pattern
},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
src_list = accepted['src']
dst = accepted['dst'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
status = returnvalues.OK
abs_dest = base_dir + dst
dst_list = glob.glob(abs_dest)
if not dst_list:
# New destination?
if not glob.glob(os.path.dirname(abs_dest)):
output_objects.append({
'object_type': 'error_text',
'text': 'Illegal dst path provided!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
else:
dst_list = [abs_dest]
# Use last match in case of multiple matches
dest = dst_list[-1]
if len(dst_list) > 1:
output_objects.append({
'object_type':
'warning',
'text':
'dst (%s) matches multiple targets - using last: %s' % (dst, dest)
})
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
logger.warning('%s tried to %s to restricted path %s ! (%s)' %
(client_id, op_name, abs_dest, dst))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % dst
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_dest))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot move to "%s": inside a read-only location!' % relative_dest
})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in src_list:
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: no such file or directory! %s' % (op_name, pattern)
})
status = returnvalues.CLIENT_ERROR
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({
'object_type': 'file',
'name': relative_path
})
# Generally refuse handling symlinks including root vgrid shares
if os.path.islink(abs_path):
output_objects.append({
'object_type':
'warning',
'text':
"""You're not allowed to
move entire special folders like %s shared folders!""" %
configuration.site_vgrid_label
})
status = returnvalues.CLIENT_ERROR
continue
# Additionally refuse operations on inherited subvgrid share roots
elif in_vgrid_share(configuration, abs_path) == relative_path:
output_objects.append({
'object_type':
'warning',
'text':
"""You're not allowed to
move entire %s shared folders!""" % configuration.site_vgrid_label
})
status = returnvalues.CLIENT_ERROR
continue
elif os.path.realpath(abs_path) == os.path.realpath(base_dir):
logger.error("%s: refusing move home dir: %s" %
(op_name, abs_path))
output_objects.append({
'object_type':
'warning',
'text':
"You're not allowed to move your entire home directory!"
})
status = returnvalues.CLIENT_ERROR
continue
if not check_write_access(abs_path):
logger.warning('%s called without write access: %s' %
(op_name, abs_path))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot move "%s": inside a read-only location!' % pattern
})
status = returnvalues.CLIENT_ERROR
continue
# If destination is a directory the src should be moved in there
# Move with existing directory as target replaces the directory!
abs_target = abs_dest
if os.path.isdir(abs_target):
if os.path.samefile(abs_target, abs_path):
output_objects.append({
'object_type':
'warning',
'text':
"Cannot move '%s' to a subdirectory of itself!" %
relative_path
})
status = returnvalues.CLIENT_ERROR
continue
abs_target = os.path.join(abs_target,
os.path.basename(abs_path))
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
'moved', [relative_path, relative_dest])
shutil.move(abs_path, abs_target)
logger.info('%s %s %s done' % (op_name, abs_path, abs_target))
except Exception as exc:
if not isinstance(exc, GDPIOLogError):
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'moved', [relative_path, relative_dest],
failed=True,
details=exc)
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False,
op_menu=client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input(
user_arguments_dict,
defaults,
output_objects,
allow_rejects=False,
# NOTE: path can use wildcards, current_dir cannot
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
current_dir = accepted['current_dir'][-1].lstrip('/')
share_id = accepted['share_id'][-1]
show_dest = accepted['with_dest'][0].lower() == 'true'
status = returnvalues.OK
# NOTE: in contrast to 'ls' we never include write operations here
read_mode, write_mode = True, False
visibility_mods = '''
.%(main_class)s .enable_write { display: none; }
.%(main_class)s .disable_read { display: none; }
.%(main_class)s .if_full { display: none; }
'''
# Either authenticated user client_id set or sharelink ID
if client_id:
user_id = client_id
target_dir = client_id_dir(client_id)
base_dir = configuration.user_home
redirect_name = configuration.site_user_redirect
redirect_path = redirect_name
id_args = ''
root_link_name = 'USER HOME'
main_class = "user_expand"
page_title = 'User Files - Path Expansion'
userstyle = True
widgets = True
elif share_id:
try:
(share_mode, _) = extract_mode_id(configuration, share_id)
except ValueError as err:
logger.error('%s called with invalid share_id %s: %s' %
(op_name, share_id, err))
output_objects.append(
{'object_type': 'error_text', 'text': 'Invalid sharelink ID: %s' % share_id})
return (output_objects, returnvalues.CLIENT_ERROR)
# TODO: load and check sharelink pickle (currently requires client_id)
# then include shared by %(owner)s on page header
user_id = 'anonymous user through share ID %s' % share_id
target_dir = os.path.join(share_mode, share_id)
base_dir = configuration.sharelink_home
redirect_name = 'share_redirect'
redirect_path = os.path.join(redirect_name, share_id)
id_args = 'share_id=%s;' % share_id
root_link_name = '%s' % share_id
main_class = "sharelink_expand"
page_title = 'Shared Files - Path Expansion'
userstyle = False
widgets = False
else:
logger.error('%s called without proper auth: %s' % (op_name, accepted))
output_objects.append({'object_type': 'error_text', 'text': 'Authentication is missing!'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
visibility_toggle = '''
<style>
%s
</style>
''' % (visibility_mods % {'main_class': main_class})
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(base_dir, target_dir)) + os.sep
if not os.path.isdir(base_dir):
logger.error('%s called on missing base_dir: %s' % (op_name, base_dir))
output_objects.append({'object_type': 'error_text', 'text': 'No such %s!' % page_title.lower()
})
return (output_objects, returnvalues.CLIENT_ERROR)
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = page_title
title_entry['skipwidgets'] = not widgets
title_entry['skipuserstyle'] = not userstyle
fill_helpers = {'dest_dir': current_dir + os.sep, 'share_id': share_id,
'flags': flags, 'tmp_flags': flags, 'long_set':
long_list(flags), 'recursive_set': recursive(flags),
'all_set': all(flags)}
add_import, add_init, add_ready = '', '', ''
title_entry['style']['advanced'] += '''
%s
''' % visibility_toggle
title_entry['script']['advanced'] += add_import
title_entry['script']['init'] += add_init
title_entry['script']['ready'] += add_ready
title_entry['script']['body'] = ' class="%s"' % main_class
output_objects.append({'object_type': 'header', 'text': page_title})
# Shared URL helpers
ls_url_template = 'ls.py?%scurrent_dir=%%(rel_dir_enc)s;flags=%s' % \
(id_args, flags)
redirect_url_template = '/%s/%%(rel_path_enc)s' % redirect_path
location_pre_html = """
<div class='files'>
<table class='files'>
<tr class=title><td class=centertext>
Working directory:
</td></tr>
<tr><td class='centertext'>
"""
output_objects.append(
{'object_type': 'html_form', 'text': location_pre_html})
# Use current_dir nav location links
for pattern in pattern_list[:1]:
links = []
links.append({'object_type': 'link', 'text': root_link_name,
'destination': ls_url_template % {'rel_dir_enc': '.'}})
prefix = ''
parts = os.path.normpath(current_dir).split(os.sep)
for i in parts:
if i == ".":
continue
prefix = os.path.join(prefix, i)
links.append({'object_type': 'link', 'text': i,
'destination': ls_url_template %
{'rel_dir_enc': quote(prefix)}})
output_objects.append(
{'object_type': 'multilinkline', 'links': links, 'sep': ' %s ' % os.sep})
location_post_html = """
</td></tr>
</table>
</div>
<br />
"""
output_objects.append(
{'object_type': 'html_form', 'text': location_post_html})
dir_listings = []
output_objects.append({
'object_type': 'dir_listings',
'dir_listings': dir_listings,
'flags': flags,
'redirect_name': redirect_name,
'redirect_path': redirect_path,
'share_id': share_id,
'ls_url_template': ls_url_template,
'rm_url_template': '',
'rmdir_url_template': '',
'editor_url_template': '',
'redirect_url_template': redirect_url_template,
'show_dest': show_dest,
})
first_match = None
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
current_path = os.path.normpath(os.path.join(base_dir, current_dir))
unfiltered_match = glob.glob(current_path + os.sep + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (user_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
if not first_match:
first_match = abs_path
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
if abs_path + os.sep == base_dir:
relative_path = '.'
else:
relative_path = abs_path.replace(base_dir, '')
entries = []
dir_listing = {
'object_type': 'dir_listing',
'relative_path': relative_path,
'entries': entries,
'flags': flags,
}
dest = ''
if show_dest:
if os.path.isfile(abs_path):
dest = os.path.basename(abs_path)
elif recursive(flags):
# references to '.' or similar are stripped by abspath
if abs_path + os.sep == base_dir:
dest = ''
else:
# dest = os.path.dirname(abs_path).replace(base_dir, "")
dest = os.path.basename(abs_path) + os.sep
handle_expand(configuration, output_objects, entries, base_dir,
abs_path, flags, dest, 0, show_dest)
dir_listings.append(dir_listing)
output_objects.append({'object_type': 'html_form', 'text': """
<div class='files disable_read'>
<form method='get' action='ls.py'>
<table class='files'>
<tr class=title><td class=centertext>
Filter paths (wildcards like * and ? are allowed)
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
<input type='text' name='path' value='' />
<input type='submit' value='Filter' />
</td></tr>
</table>
</form>
</div>
""" % fill_helpers})
# Short/long format buttons
fill_helpers['tmp_flags'] = flags + 'l'
htmlform = """
<table class='files if_full'>
<tr class=title><td class=centertext colspan=4>
File view options
</td></tr>
<tr><td colspan=4><br /></td></tr>
<tr class=title><td>Parameter</td><td>Setting</td><td>Enable</td><td>Disable</td></tr>
<tr><td>Long format</td><td>
%(long_set)s</td><td>
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />" % entry
fill_helpers['tmp_flags'] = flags.replace('l', '')
htmlform += """
<input type='submit' value='On' /><br />
</form>
</td><td>
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />" % entry
htmlform += """
<input type='submit' value='Off' /><br />
</form>
</td></tr>
"""
# Recursive output
fill_helpers['tmp_flags'] = flags + 'r'
htmlform += """
<!-- Non-/recursive list buttons -->
<tr><td>Recursion</td><td>
%(recursive_set)s</td><td>""" % fill_helpers
htmlform += """
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />" % entry
fill_helpers['tmp_flags'] = flags.replace('r', '')
htmlform += """
<input type='submit' value='On' /><br />
</form>
</td><td>
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />"\
% entry
htmlform += """
<input type='submit' value='Off' /><br />
</form>
</td></tr>
"""
htmlform += """
<!-- Show dot files buttons -->
<tr><td>Show hidden files</td><td>
%(all_set)s</td><td>""" % fill_helpers
fill_helpers['tmp_flags'] = flags + 'a'
htmlform += """
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />" % entry
fill_helpers['tmp_flags'] = flags.replace('a', '')
htmlform += """
<input type='submit' value='On' /><br />
</form>
</td><td>
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />" % entry
htmlform += """
<input type='submit' value='Off' /><br />
</form>
</td></tr>
</table>
"""
# show flag buttons after contents to limit clutter
output_objects.append({'object_type': 'html_form', 'text': htmlform})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path can use wildcards
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
name_pattern = accepted['name'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
# Shared URL helpers
id_args = ''
redirect_name = configuration.site_user_redirect
redirect_path = redirect_name
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
ls_url_template = 'ls.py?%scurrent_dir=%%(rel_dir_enc)s;flags=%s' % \
(id_args, flags)
csrf_token = make_csrf_token(configuration, form_method, 'rm', client_id,
csrf_limit)
rm_url_template = 'rm.py?%spath=%%(rel_path_enc)s;%s=%s' % \
(id_args, csrf_field, csrf_token)
rmdir_url_template = 'rm.py?%spath=%%(rel_path_enc)s;flags=r;%s=%s' % \
(id_args, csrf_field, csrf_token)
editor_url_template = 'editor.py?%spath=%%(rel_path_enc)s' % id_args
redirect_url_template = '/%s/%%(rel_path_enc)s' % redirect_path
dir_listings = []
output_objects.append({
'object_type': 'dir_listings',
'dir_listings': dir_listings,
'flags': flags,
'ls_url_template': ls_url_template,
'rm_url_template': rm_url_template,
'rmdir_url_template': rmdir_url_template,
'editor_url_template': editor_url_template,
'redirect_url_template': redirect_url_template,
})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
entries = []
dir_listing = {
'object_type': 'dir_listing',
'relative_path': relative_path,
'entries': entries,
'flags': flags,
}
dir_listings.append(dir_listing)
try:
for (root, dirs, files) in walk(abs_path):
for filename in fnmatch.filter(files, name_pattern):
# IMPORTANT: this join always yields abs expanded path
abs_path = os.path.join(root, filename)
relative_path = abs_path.replace(base_dir, '')
if not valid_user_path(configuration, abs_path,
base_dir, True):
continue
file_with_dir = relative_path
file_obj = {
'object_type': 'direntry',
'type': 'file',
'name': filename,
'rel_path': file_with_dir,
'rel_path_enc': quote(file_with_dir),
'rel_dir_enc':
quote(os.path.dirname(file_with_dir)),
# NOTE: file_with_dir is kept for backwards compliance
'file_with_dir': file_with_dir,
'flags': flags,
'special': '',
}
entries.append(file_obj)
except Exception as exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if verbose(flags):
output_objects.append({
'object_type': 'file_output',
'path': relative_path,
'lines': output_lines
})
else:
output_objects.append({
'object_type': 'file_output',
'lines': output_lines
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
status = returnvalues.OK
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: src can use wildcards, dst cannot
typecheck_overrides={'src': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
job_ids = accepted['job_id']
action = accepted['action'][-1]
src = accepted['src']
dst = accepted['dst'][-1]
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = '%s live I/O' % configuration.short_title
add_import, add_init, add_ready = '', '', ''
add_init += '''
var fields = 1;
var max_fields = 20;
var src_input = "<input class='fillwidth' type=text name=src value='' /><br />";
function addSource() {
if (fields < max_fields) {
$("#srcfields").append(src_input);
fields += 1;
} else {
alert("Maximum " + max_fields + " source fields allowed!");
}
}
'''
title_entry['script']['advanced'] += add_import
title_entry['script']['init'] += add_init
title_entry['script']['ready'] += add_ready
output_objects.append({
'object_type': 'header',
'text': 'Request live communication with jobs'
})
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not action in valid_actions:
output_objects.append({
'object_type':
'error_text',
'text':
'Invalid action "%s" (supported: %s)' %
(action, ', '.join(valid_actions))
})
return (output_objects, returnvalues.CLIENT_ERROR)
if action in post_actions:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not job_ids or action in interactive_actions:
job_id = ''
if job_ids:
job_id = job_ids[-1]
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
fill_helpers = {
'job_id': job_id,
'form_method': form_method,
'csrf_field': csrf_field,
'csrf_limit': csrf_limit
}
target_op = 'liveio'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token})
output_objects.append({
'object_type':
'text',
'text':
'''
Fill in the live I/O details below to request communication with a running
job.
Job ID can be a full ID or a wild card pattern using "*" and "?" to match one
or more of your job IDs.
Use send output without source and destination paths to request upload of the
default stdio files from the job on the resource to the associated job_output
directory in your MiG home.
Destination is a always handled as a directory path to put source files into.
Source and destination paths are always taken relative to the job execution
directory on the resource and your MiG home respectively.
'''
})
html = '''
<table class="liveio">
<tr>
<td>
<form method="%(form_method)s" action="%(target_op)s.py">
<table class="liveio">
<tr><td class=centertext>
</td></tr>
<tr><td>
Action:<br />
<input type="hidden" name="%(csrf_field)s" value="%(csrf_token)s" />
<input type=radio name=action checked value="send" />send output
<input type=radio name=action value="get" />get input
</td></tr>
<tr><td>
Job ID:<br />
<input class="fillwidth" type=text name=job_id value="%(job_id)s" />
</td></tr>
<tr><td>
Source path(s):<br />
<div id="srcfields">
<input class="fillwidth" type=text name=src value="" /><br />
</div>
<input id="addsrcbutton" type="button" onclick="addSource(); return false;"
value="Add another source field" />
</td></tr>
<tr><td>
Destination path:<br />
<input class="fillwidth" type=text name=dst value="" />
</td></tr>
<tr><td>
<input type="submit" value="Send request" />
</td></tr>
</table>
</form>
</td>
</tr>
</table>
''' % fill_helpers
output_objects.append({'object_type': 'html_form', 'text': html})
output_objects.append({
'object_type':
'text',
'text':
'''
Further live job control is avalable through your personal message queues.
They provide a basic interface for centrally storing messages under your grid
account and can be used to pass messages between jobs or for orchestrating
jobs before and during execution.
'''
})
output_objects.append({
'object_type': 'link',
'destination': 'mqueue.py',
'text': 'Message queue interface'
})
return (output_objects, returnvalues.OK)
elif action in ['get', 'receive', 'input']:
action = 'get'
action_desc = 'will be downloaded to the job on the resource'
elif action in ['put', 'send', 'output']:
action = 'send'
action_desc = 'will be uploaded from the job on the resource'
else:
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid live io action: %s' % action
})
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append({
'object_type':
'text',
'text':
'Requesting live I/O for %s' % ', '.join(job_ids)
})
if action == 'get' and (not src or not dst):
output_objects.append({
'object_type':
'error_text',
'text':
'src and dst parameters required for live input'
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Automatic fall back to stdio files if output with no path provided
if src:
src_text = 'The files ' + ' '.join(src)
else:
src_text = 'The job stdio files'
if dst:
dst_text = 'the ' + dst + ' directory'
else:
dst_text = 'the corresponding job_output directory'
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
filelist = []
for job_id in job_ids:
job_id = job_id.strip()
# is job currently being executed?
# Backward compatibility - all_jobs keyword should match all jobs
if job_id == all_jobs:
job_id = '*'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + job_id + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning("%s tried to %s restricted path %s ! (%s)" %
(client_id, op_name, abs_path, job_id))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match....
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % job_id
})
else:
filelist += match
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
job_dict = unpickle(filepath, logger)
if not job_dict:
status = returnvalues.CLIENT_ERROR
output_objects.append({
'object_type':
'error_text',
'text':
('You can only list status of your own jobs. '
'Please verify that you submitted the mRSL file '
'with job id "%s" (Could not unpickle mRSL file %s)') %
(job_id, filepath)
})
continue
if job_dict['STATUS'] != 'EXECUTING':
output_objects.append({
'object_type':
'text',
'text':
'Job %s is not currently being executed! Job status: %s' %
(job_id, job_dict['STATUS'])
})
continue
if job_dict['UNIQUE_RESOURCE_NAME'] == 'ARC':
output_objects.append({
'object_type':
'text',
'text':
'Job %s is submitted to ARC, details are not available!' %
job_id
})
continue
last_live_update_dict = {}
last_live_update_file = configuration.mig_system_files + os.sep\
+ job_id + '.last_live_update'
if os.path.isfile(last_live_update_file):
last_live_update_dict_unpickled = \
unpickle(last_live_update_file, logger)
if not last_live_update_dict_unpickled:
output_objects.append({
'object_type':
'error_text',
'text':
'Could not unpickle %s - skipping request!' %
last_live_update_file
})
continue
if 'LAST_LIVE_UPDATE_REQUEST_TIMESTAMP' not in last_live_update_dict_unpickled:
output_objects.append({
'object_type':
'error_text',
'text':
'Could not find needed key in %s.' % last_live_update_file
})
continue
last_live_update_request = \
last_live_update_dict_unpickled['LAST_LIVE_UPDATE_REQUEST_TIMESTAMP'
]
difference = datetime.datetime.now() - last_live_update_request
try:
min_delay = \
int(configuration.min_seconds_between_live_update_requests)
except:
min_delay = 30
if difference.seconds < min_delay:
output_objects.append({
'object_type':
'error_text',
'text':
('Request not allowed, you must wait at least '
'%s seconds between live update requests!') % min_delay
})
continue
# save this request to file to avoid DoS from a client request loop.
last_live_update_dict['LAST_LIVE_UPDATE_REQUEST_TIMESTAMP'] = \
datetime.datetime.now()
pickle_ret = pickle(last_live_update_dict, last_live_update_file,
logger)
if not pickle_ret:
output_objects.append({
'object_type':
'error_text',
'text':
'Error saving live io request timestamp to last_live_update '
'file, request not sent!'
})
continue
# #
# ## job is being executed right now, send live io request to frontend
# #
# get resource_config, needed by scp_file_to_resource
# (res_status, resource_config) = get_resource_configuration(
# resource_home, unique_resource_name, logger)
resource_config = job_dict['RESOURCE_CONFIG']
(res_status, exe) = get_resource_exe(resource_config, job_dict['EXE'],
logger)
if not res_status:
output_objects.append({
'object_type':
'error_text',
'text':
'Could not get exe configuration for job %s' % job_id
})
continue
local_file = '%s.%supdate' % (job_dict['LOCALJOBNAME'], action)
if not os.path.exists(local_file):
# create
try:
filehandle = open(local_file, 'w')
filehandle.write('job_id ' + job_dict['JOB_ID'] + '\n')
filehandle.write('localjobname ' + job_dict['LOCALJOBNAME'] +
'\n')
filehandle.write('execution_user ' + exe['execution_user'] +
'\n')
filehandle.write('execution_node ' + exe['execution_node'] +
'\n')
filehandle.write('execution_dir ' + exe['execution_dir'] +
'\n')
filehandle.write('target liveio\n')
# Leave defaults src and dst to FE script if not provided
if src:
filehandle.write('source ' + ' '.join(src) + '\n')
if dst:
filehandle.write('destination ' + dst + '\n')
# Backward compatible test for shared_fs - fall back to scp
if 'shared_fs' in exe and exe['shared_fs']:
filehandle.write('copy_command cp\n')
filehandle.write('copy_frontend_prefix \n')
filehandle.write('copy_execution_prefix \n')
else:
filehandle.write('copy_command scp -B\n')
filehandle.write(
'copy_frontend_prefix ${frontend_user}@${frontend_node}:\n'
)
filehandle.write(
'copy_execution_prefix ${execution_user}@${execution_node}:\n'
)
filehandle.write('### END OF SCRIPT ###\n')
filehandle.close()
except Exception as exc:
pass
if not os.path.exists(local_file):
output_objects.append({
'object_type':
'error_text',
'text':
'.%supdate file not available on %s server' %
(action, configuration.short_title)
})
continue
scp_status = copy_file_to_resource(
local_file, '%s.%supdate' % (job_dict['LOCALJOBNAME'], action),
resource_config, logger)
if not scp_status:
output_objects.append({
'object_type':
'error_text',
'text':
'Error sending request for live io to resource!'
})
continue
else:
output_objects.append({
'object_type':
'text',
'text':
'Request for live io was successfully sent to the resource!'
})
output_objects.append({
'object_type':
'text',
'text':
'%s %s and should become available in %s in a minute.' %
(src_text, action_desc, dst_text)
})
if action == 'send':
if not dst:
target_path = '%s/%s/*' % (job_output_dir, job_id)
else:
target_path = dst
enc_url = 'ls.py?path=%s' % quote(target_path)
output_objects.append({
'object_type': 'link',
'destination': enc_url,
'text': 'View uploaded files'
})
else:
enc_url = 'ls.py?path='
enc_url += ';path='.join([quote(i) for i in src])
output_objects.append({
'object_type': 'link',
'destination': enc_url,
'text': 'View files for download'
})
try:
os.remove(local_file)
except Exception as exc:
pass
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
valid_langs = {'sh': 'shell', 'python': 'python'}
valid_flavors = {'user': '******',
'resource': 'vgridscriptgen'}
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
langs = accepted['lang']
flavor_list = accepted['flavor']
sh_cmd = accepted['sh_cmd'][-1]
python_cmd = accepted['python_cmd'][-1]
script_dir = accepted['script_dir'][-1]
flavors = []
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Script generator'
output_objects.append(
{'object_type': 'header', 'text': 'Script generator'})
status = returnvalues.OK
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if 'h' in flags:
output_objects = usage(output_objects, valid_langs,
valid_flavors)
return (output_objects, status)
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append(
{'object_type': 'error_text', 'text': '''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Filter out any invalid flavors to avoid illegal filenames, etc.
for f in flavor_list:
if f in valid_flavors.keys():
flavors.append(f)
# Default to user scripts
if not flavors:
if flavor_list:
output_objects.append({'object_type': 'text', 'text': 'No valid flavors specified - falling back to user scripts'
})
flavors = ['user']
if not langs or keyword_all in langs:
# Add new languages here
languages = [(userscriptgen.sh_lang, sh_cmd, userscriptgen.sh_ext),
(userscriptgen.python_lang, python_cmd,
userscriptgen.python_ext)]
else:
languages = []
# check arguments
for lang in langs:
if lang == 'sh':
interpreter = sh_cmd
extension = userscriptgen.sh_ext
elif lang == 'python':
interpreter = python_cmd
extension = userscriptgen.python_ext
else:
output_objects.append({'object_type': 'warning', 'text': 'Unknown script language: %s - ignoring!'
% lang})
continue
languages.append((lang, interpreter, extension))
if not languages:
output_objects.append({'object_type': 'error_text', 'text': 'No valid languages specified - aborting script generation'
})
return (output_objects, returnvalues.CLIENT_ERROR)
for flavor in flavors:
if not script_dir or script_dir == keyword_auto:
# Generate scripts in a "unique" destination directory
# gmtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
# tm_sec, tm_wday, tm_yday, tm_isdst)
now = time.gmtime()
timestamp = '%.2d%.2d%.2d-%.2d%.2d%.2d' % (
now[2],
now[1],
now[0],
now[3],
now[4],
now[5],
)
script_dir = '%s-%s-scripts-%s' % (configuration.short_title,
flavor, timestamp)
else:
# Avoid problems from especially trailing slash (zip recursion)
script_dir = script_dir.strip(os.sep)
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dir = os.path.abspath(os.path.join(base_dir, script_dir))
if not valid_user_path(configuration, abs_dir, base_dir, True):
# out of bounds
output_objects.append({'object_type': 'error_text', 'text': "You're not allowed to work in %s!"
% script_dir})
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_dir, script_dir))
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(abs_dir):
try:
os.mkdir(abs_dir)
except Exception as exc:
output_objects.append({'object_type': 'error_text',
'text': 'Failed to create destination directory (%s) - aborting script generation'
% exc})
return (output_objects, returnvalues.SYSTEM_ERROR)
for (lang, _, _) in languages:
output_objects.append({'object_type': 'text', 'text': 'Generating %s %s scripts in the %s subdirectory of your %s home directory'
% (lang, flavor, script_dir, configuration.short_title)})
logger.debug('generate %s scripts in %s' % (flavor, abs_dir))
# Generate all scripts
if flavor == 'user':
for op in userscriptgen.script_ops:
generator = 'userscriptgen.generate_%s' % op
eval(generator)(configuration, languages, abs_dir)
if userscriptgen.shared_lib:
userscriptgen.generate_lib(configuration, userscriptgen.script_ops,
languages, abs_dir)
if userscriptgen.test_script:
userscriptgen.generate_test(configuration, languages, abs_dir)
elif flavor == 'resource':
for op in vgridscriptgen.script_ops_single_arg:
vgridscriptgen.generate_single_argument(configuration, op[0], op[1],
languages, abs_dir)
for op in vgridscriptgen.script_ops_single_upload_arg:
vgridscriptgen.generate_single_argument_upload(configuration, op[0],
op[1], op[2],
languages, abs_dir)
for op in vgridscriptgen.script_ops_two_args:
vgridscriptgen.generate_two_arguments(configuration, op[0], op[1],
op[2], languages, abs_dir)
for op in vgridscriptgen.script_ops_ten_args:
vgridscriptgen.generate_ten_arguments(configuration, op[0], op[1],
op[2], op[3], op[4], op[5],
op[6], op[7], op[8], op[9],
op[10], languages, abs_dir)
else:
output_objects.append(
{'object_type': 'warning_text', 'text': 'Unknown flavor: %s' % flavor})
continue
# Always include license conditions file
userscriptgen.write_license(configuration, abs_dir)
output_objects.append({'object_type': 'text', 'text': '... Done'
})
output_objects.append({'object_type': 'text', 'text': '%s %s scripts are now available in your %s home directory:'
% (configuration.short_title, flavor, configuration.short_title)})
output_objects.append({'object_type': 'link', 'text': 'View directory',
'destination': 'fileman.py?path=%s/' % script_dir})
# Create zip from generated dir
output_objects.append({'object_type': 'text', 'text': 'Generating zip archive of the %s %s scripts'
% (configuration.short_title, flavor)})
script_zip = script_dir + '.zip'
dest_zip = '%s%s' % (base_dir, script_zip)
logger.debug('packing generated scripts from %s in %s' % (abs_dir,
dest_zip))
# Force compression
zip_file = zipfile.ZipFile(dest_zip, 'w', zipfile.ZIP_DEFLATED)
# Directory write is not supported - add each file manually
for script in os.listdir(abs_dir):
zip_file.write(abs_dir + os.sep + script, script_dir
+ os.sep + script)
# Preserve executable flag in accordance with:
# http://mail.python.org/pipermail/pythonmac-sig/2005-March/013491.html
for zinfo in zip_file.filelist:
zinfo.create_system = 3
zip_file.close()
# Verify CRC
zip_file = zipfile.ZipFile(dest_zip, 'r')
err = zip_file.testzip()
zip_file.close()
if err:
output_objects.append({'object_type': 'error_text', 'text': 'Zip file integrity check failed! (%s)'
% err})
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'text', 'text': '... Done'
})
output_objects.append({'object_type': 'text', 'text': 'Zip archive of the %s %s scripts are now available in your %s home directory'
% (configuration.short_title, flavor, configuration.short_title)})
output_objects.append({'object_type': 'link', 'text': 'Download zip archive %s' % script_zip, 'destination': os.path.join('..', client_dir,
script_zip)})
output_objects.append({'object_type': 'upgrade_info', 'text': '''
You can upgrade from an existing user scripts folder with the commands:''',
'commands': ["./migget.sh '%s' ../" % script_zip,
"cd ..", "unzip '%s'" % script_zip,
"cd '%s'" % script_dir]
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False,
op_menu=client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input(user_arguments_dict,
defaults,
output_objects,
allow_rejects=False)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
queue = accepted['queue'][-1]
action = accepted['action'][-1]
iosessionid = accepted['iosessionid'][-1]
msg = accepted['msg'][-1]
msg_id = accepted['msg_id'][-1]
# Web format for cert access and no header for SID access
if client_id:
output_objects.append({
'object_type': 'header',
'text': 'Message queue %s' % action
})
else:
output_objects.append({'object_type': 'start'})
# Always return at least a basic file_output entry
file_entry = {
'object_type': 'file_output',
'lines': [],
'wrap_binary': True,
'wrap_targets': ['lines']
}
if not action in valid_actions:
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid action "%s" (supported: %s)' % \
(action, ', '.join(valid_actions))})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
if action in post_actions:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Find user home from session or certificate
if iosessionid:
client_home = os.path.realpath(
os.path.join(configuration.webserver_home, iosessionid))
client_dir = os.path.basename(client_home)
elif client_id:
client_dir = client_id_dir(client_id)
else:
output_objects.append({
'object_type':
'error_text',
'text':
'Either certificate or session ID is required'
})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if not os.path.isdir(base_dir):
output_objects.append({
'object_type': 'error_text',
'text': 'No matching session or user home!'
})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
mqueue_base = os.path.join(base_dir, mqueue_prefix) + os.sep
default_queue_dir = os.path.join(mqueue_base, default_mqueue)
# Create mqueue base and default queue dir if missing
if not os.path.exists(default_queue_dir):
try:
os.makedirs(default_queue_dir)
except:
pass
# IMPORTANT: path must be expanded to abs for proper chrooting
queue_path = os.path.abspath(os.path.join(mqueue_base, queue))
if not valid_user_path(configuration, queue_path, mqueue_base):
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid queue name: "%s"' % queue
})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
lock_path = os.path.join(mqueue_base, lock_name)
lock_handle = open(lock_path, 'a')
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
status = returnvalues.OK
if action == "interactive":
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
fill_helpers = {
'queue': queue,
'msg': msg,
'form_method': form_method,
'csrf_field': csrf_field,
'csrf_limit': csrf_limit,
}
target_op = 'mqueue'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token})
output_objects.append({
'object_type':
'text',
'text':
'''
Fill in the fields below to control and access your personal message queues.
Jobs can receive from and send to the message queues during execution, and use
them as a means of job inter-communication. Expect message queue operations to
take several seconds on the resources, however. That is, use it for tasks like
orchestrating long running jobs, and not for low latency communication.
'''
})
html = '''
<form name="mqueueform" method="%(form_method)s" action="%(target_op)s.py">
<table class="mqueue">
<tr><td class=centertext>
</td></tr>
<tr><td>
Action:<br />
<input type="hidden" name="%(csrf_field)s" value="%(csrf_token)s" />
<input type=radio name=action value="create" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />create queue
<input type=radio name=action checked value="send" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=false;" />send message to queue
<input type=radio name=action value="receive" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />receive message from queue
<input type=radio name=action value="remove" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />remove queue
<input type=radio name=action value="listqueues" onclick="javascript: document.mqueueform.queue.disabled=true; document.mqueueform.msg.disabled=true;" />list queues
<input type=radio name=action value="listmessages" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />list messages
<input type=radio name=action value="show" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />show message
</td></tr>
<tr><td>
Queue:<br />
<input class="fillwidth" type=text name=queue value="%(queue)s" />
</td></tr>
<tr><td>
<div id="msgfieldf">
<input class="fillwidth" type=text name=msg value="%(msg)s" /><br />
</div>
</td></tr>
<tr><td>
<input type="submit" value="Apply" />
</td></tr>
</table>
</form>
''' % fill_helpers
output_objects.append({'object_type': 'html_form', 'text': html})
output_objects.append({
'object_type':
'text',
'text':
'''
Further live job control is avalable through the live I/O interface.
They provide a basic interface for centrally managing input and output files
for active jobs.
'''
})
output_objects.append({
'object_type': 'link',
'destination': 'liveio.py',
'text': 'Live I/O interface'
})
return (output_objects, returnvalues.OK)
elif action == 'create':
try:
os.mkdir(queue_path)
output_objects.append({
'object_type': 'text',
'text': 'New "%s" queue created' % queue
})
except Exception as err:
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not create "%s" queue: "%s"' % \
(queue, err)})
status = returnvalues.CLIENT_ERROR
elif action == 'remove':
try:
for entry in os.listdir(queue_path):
os.remove(os.path.join(queue_path, entry))
os.rmdir(queue_path)
output_objects.append({
'object_type': 'text',
'text': 'Existing "%s" queue removed' % queue
})
except Exception as err:
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not remove "%s" queue: "%s"' % \
(queue, err)})
status = returnvalues.CLIENT_ERROR
elif action == 'send':
try:
if not msg_id:
msg_id = "%.0f" % time.time()
msg_path = os.path.join(queue_path, msg_id)
msg_fd = open(msg_path, 'w')
msg_fd.write(msg)
msg_fd.close()
output_objects.append({
'object_type': 'text',
'text': 'Message sent to "%s" queue' % queue
})
except Exception as err:
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not send to "%s" queue: "%s"' % \
(queue, err)})
status = returnvalues.CLIENT_ERROR
elif action == 'receive':
try:
if not msg_id:
messages = os.listdir(queue_path)
messages.sort()
if messages:
msg_id = messages[0]
if msg_id:
message_path = os.path.join(queue_path, msg_id)
message_fd = open(message_path, 'r')
message = message_fd.readlines()
message_fd.close()
os.remove(message_path)
file_entry['path'] = os.path.basename(message_path)
else:
message = [mqueue_empty]
# Update file_output entry for raw data with output_format=file
file_entry['lines'] = message
except Exception as err:
output_objects.append({
'object_type':
'error_text',
'text':
'Could not receive from "%s" queue: "%s"' % (queue, err)
})
status = returnvalues.CLIENT_ERROR
elif action == 'show':
try:
if not msg_id:
messages = os.listdir(queue_path)
messages.sort()
if messages:
msg_id = messages[0]
if msg_id:
message_path = os.path.join(queue_path, msg_id)
message_fd = open(message_path, 'r')
message = message_fd.readlines()
message_fd.close()
file_entry['path'] = os.path.basename(message_path)
else:
message = [mqueue_empty]
# Update file_output entry for raw data with output_format=file
file_entry['lines'] = message
except Exception as err:
output_objects.append({
'object_type':
'error_text',
'text':
'Could not show %s from "%s" queue: "%s"' %
(msg_id, queue, err)
})
status = returnvalues.CLIENT_ERROR
elif action == 'listmessages':
try:
messages = os.listdir(queue_path)
messages.sort()
output_objects.append({'object_type': 'list', 'list': messages})
except Exception as err:
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not list "%s" queue: "%s"' % \
(queue, err)})
status = returnvalues.CLIENT_ERROR
elif action == 'listqueues':
try:
queues = [i for i in os.listdir(mqueue_base) if \
os.path.isdir(os.path.join(mqueue_base, i))]
queues.sort()
output_objects.append({'object_type': 'list', 'list': queues})
except Exception as err:
output_objects.append({
'object_type': 'error_text',
'text': 'Could not list queues: "%s"' % err
})
status = returnvalues.CLIENT_ERROR
else:
output_objects.append({
'object_type': 'error_text',
'text': 'Unexpected mqueue action: "%s"' % action
})
status = returnvalues.SYSTEM_ERROR
lock_handle.close()
output_objects.append(file_entry)
output_objects.append({
'object_type':
'link',
'destination':
'mqueue.py?queue=%s;msg=%s' % (queue, msg),
'text':
'Back to message queue interaction'
})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = accepted['flags']
patterns = accepted['job_id']
if not configuration.site_enable_jobs:
output_objects.append({'object_type': 'error_text', 'text':
'''Job execution is not enabled on this system'''})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
mrsl_keywords_dict = get_keywords_dict(configuration)
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in patterns:
# Add file extension
pattern += '.mRSL'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found', 'name'
: pattern})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
try:
mrsl_dict = unpickle(abs_path, logger)
if not mrsl_dict:
raise Exception('could not load job mRSL')
for (key, val) in mrsl_dict.items():
if not key in mrsl_keywords_dict.keys():
continue
if not val:
continue
output_lines.append('::%s::\n' % key)
if 'multiplestrings' == mrsl_keywords_dict[key]['Type']:
for line in val:
output_lines.append('%s\n' % line)
elif 'multiplekeyvalues' == mrsl_keywords_dict[key]['Type']:
for (left, right) in val:
output_lines.append('%s=%s\n' % (left, right))
else:
output_lines.append('%s\n' % val)
output_lines.append('\n')
except Exception as exc:
output_objects.append({'object_type': 'error_text', 'text'
: "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if verbose(flags):
output_objects.append({'object_type': 'file_output', 'path'
: relative_path, 'lines'
: output_lines})
else:
output_objects.append({'object_type': 'file_output', 'lines'
: output_lines})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
logger.error("jobstatus input validation failed: %s" % accepted)
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
max_jobs = int(accepted['max_jobs'][-1])
order = 'unsorted '
if sorted(flags):
order = 'sorted '
patterns = accepted['job_id']
project_names = accepted['project_name']
if len(project_names) > 0:
for project_name in project_names:
project_name_job_ids = \
get_job_ids_with_specified_project_name(client_id,
project_name, configuration.mrsl_files_dir, logger)
patterns.extend(project_name_job_ids)
if not configuration.site_enable_jobs:
output_objects.append({'object_type': 'error_text', 'text':
'''Job execution is not enabled on this system'''})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
output_objects.append({'object_type': 'header', 'text': '%s %s job status' %
(configuration.short_title, order)})
if not patterns:
output_objects.append(
{'object_type': 'error_text', 'text': 'No job_id specified!'})
return (output_objects, returnvalues.NO_SUCH_JOB_ID)
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text': '%s using flag: %s' % (op_name,
flag)})
if not os.path.isdir(base_dir):
output_objects.append(
{'object_type': 'error_text', 'text': ('You have not been created as a user on the %s server! '
'Please contact the %s team.') %
(configuration.short_title, configuration.short_title)})
return (output_objects, returnvalues.CLIENT_ERROR)
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to
# avoid leaking information about file system layout while
# allowing consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match....
if not match:
output_objects.append(
{'object_type': 'error_text', 'text': '%s: You do not have any matching job IDs!' % pattern})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
if sorted(flags):
sort(filelist)
if max_jobs > 0 and max_jobs < len(filelist):
output_objects.append(
{'object_type': 'text', 'text': 'Only showing first %d of the %d matching jobs as requested'
% (max_jobs, len(filelist))})
filelist = filelist[:max_jobs]
# Iterate through jobs and list details for each
job_list = {'object_type': 'job_list', 'jobs': []}
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
job_dict = unpickle(filepath, logger)
if not job_dict:
status = returnvalues.CLIENT_ERROR
output_objects.append(
{'object_type': 'error_text', 'text': 'No such job: %s (could not load mRSL file %s)' %
(job_id, filepath)})
continue
# Expand any job variables before use
job_dict = expand_variables(job_dict)
job_obj = {'object_type': 'job', 'job_id': job_id}
job_obj['status'] = job_dict['STATUS']
time_fields = [
'VERIFIED',
'VERIFIED_TIMESTAMP',
'RECEIVED_TIMESTAMP',
'QUEUED_TIMESTAMP',
'SCHEDULE_TIMESTAMP',
'EXECUTING_TIMESTAMP',
'FINISHED_TIMESTAMP',
'FAILED_TIMESTAMP',
'CANCELED_TIMESTAMP',
]
for name in time_fields:
if name in job_dict:
# time objects cannot be marshalled, asctime if timestamp
try:
job_obj[name.lower()] = time.asctime(job_dict[name])
except Exception as exc:
# not a time object, just add
job_obj[name.lower()] = job_dict[name]
###########################################
# ARC job status retrieval on demand:
# But we should _not_ update the status in the mRSL files, since
# other MiG code might rely on finding only valid "MiG" states.
if configuration.arc_clusters and \
job_dict.get('UNIQUE_RESOURCE_NAME', 'unset') == 'ARC' \
and job_dict['STATUS'] == 'EXECUTING':
try:
home = os.path.join(configuration.user_home, client_dir)
arcsession = arcwrapper.Ui(home)
arcstatus = arcsession.jobStatus(job_dict['EXE'])
job_obj['status'] = arcstatus['status']
except arcwrapper.ARCWrapperError as err:
logger.error('Error retrieving ARC job status: %s' %
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
except arcwrapper.NoProxyError as err:
logger.error('While retrieving ARC job status: %s' %
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
except Exception as err:
logger.error('Error retrieving ARC job status: %s' % err)
job_obj['status'] += '(Error during retrieval)'
exec_histories = []
if verbose(flags):
if 'EXECUTE' in job_dict:
command_line = '; '.join(job_dict['EXECUTE'])
if len(command_line) > 256:
job_obj['execute'] = '%s ...' % command_line[:252]
else:
job_obj['execute'] = command_line
res_conf = job_dict.get('RESOURCE_CONFIG', {})
if 'RESOURCE_ID' in res_conf:
public_id = res_conf['RESOURCE_ID']
if res_conf.get('ANONYMOUS', True):
public_id = anon_resource_id(public_id)
job_obj['resource'] = public_id
if job_dict.get('PUBLICNAME', False):
job_obj['resource'] += ' (alias %(PUBLICNAME)s)' % job_dict
if 'RESOURCE_VGRID' in job_dict:
job_obj['vgrid'] = job_dict['RESOURCE_VGRID']
if 'EXECUTION_HISTORY' in job_dict:
counter = 0
for history_dict in job_dict['EXECUTION_HISTORY']:
exec_history = \
{'object_type': 'execution_history'}
if 'QUEUED_TIMESTAMP' in history_dict:
exec_history['queued'] = \
time.asctime(history_dict['QUEUED_TIMESTAMP'
])
if 'EXECUTING_TIMESTAMP' in history_dict:
exec_history['executing'] = \
time.asctime(history_dict['EXECUTING_TIMESTAMP'
])
if 'PUBLICNAME' in history_dict:
if history_dict['PUBLICNAME']:
exec_history['resource'] = \
history_dict['PUBLICNAME']
else:
exec_history['resource'] = 'HIDDEN'
if 'RESOURCE_VGRID' in history_dict:
exec_history['vgrid'] = \
history_dict['RESOURCE_VGRID']
if 'FAILED_TIMESTAMP' in history_dict:
exec_history['failed'] = \
time.asctime(history_dict['FAILED_TIMESTAMP'
])
if 'FAILED_MESSAGE' in history_dict:
exec_history['failed_message'] = \
history_dict['FAILED_MESSAGE']
exec_histories.append(
{'execution_history': exec_history, 'count': counter})
counter += 1
if 'SCHEDULE_HINT' in job_dict:
job_obj['schedule_hint'] = job_dict['SCHEDULE_HINT']
# We should not show raw schedule_targets due to lack of anonymization
if 'SCHEDULE_TARGETS' in job_dict:
job_obj['schedule_hits'] = len(job_dict['SCHEDULE_TARGETS'])
if 'EXPECTED_DELAY' in job_dict:
# Catch None value
if not job_dict['EXPECTED_DELAY']:
job_obj['expected_delay'] = 0
else:
job_obj['expected_delay'] = int(job_dict['EXPECTED_DELAY'])
job_obj['execution_histories'] = exec_histories
if interactive(flags):
job_obj['statuslink'] = {'object_type': 'link',
'destination': 'fileman.py?path=%s/%s/'
% (job_output_dir, job_id), 'text':
'View status files'}
job_obj['mrsllink'] = {'object_type': 'link',
'destination': 'mrslview.py?job_id=%s'
% job_id,
'text': 'View parsed mRSL contents'}
if 'OUTPUTFILES' in job_dict and job_dict['OUTPUTFILES']:
# Create a single ls link with all supplied outputfiles
path_string = ''
for path in job_dict['OUTPUTFILES']:
# OUTPUTFILES is either just combo path or src dst paths
parts = path.split()
# Always take last part as destination
path_string += 'path=%s;' % parts[-1]
job_obj['outputfileslink'] = {'object_type': 'link',
'destination': 'ls.py?%s' %
path_string,
'text': 'View output files'}
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
target_op = 'resubmit'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
js_name = 'resubmit%s' % hexlify(job_id)
helper = html_post_helper(js_name, '%s.py' % target_op,
{'job_id': job_id,
csrf_field: csrf_token})
output_objects.append({'object_type': 'html_form', 'text': helper})
job_obj['resubmitlink'] = {'object_type': 'link',
'destination':
"javascript: %s();" % js_name,
'text': 'Resubmit job'}
target_op = 'jobaction'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
js_name = 'freeze%s' % hexlify(job_id)
helper = html_post_helper(js_name, '%s.py' % target_op,
{'action': 'freeze', 'job_id': job_id,
csrf_field: csrf_token})
output_objects.append({'object_type': 'html_form', 'text': helper})
job_obj['freezelink'] = {'object_type': 'link',
'destination':
"javascript: %s();" % js_name,
'text': 'Freeze job in queue'}
js_name = 'thaw%s' % hexlify(job_id)
helper = html_post_helper(js_name, '%s.py' % target_op,
{'action': 'thaw', 'job_id': job_id,
csrf_field: csrf_token})
output_objects.append({'object_type': 'html_form', 'text': helper})
job_obj['thawlink'] = {'object_type': 'link',
'destination':
"javascript: %s();" % js_name,
'text': 'Thaw job in queue'}
js_name = 'cancel%s' % hexlify(job_id)
helper = html_post_helper(js_name, '%s.py' % target_op,
{'action': 'cancel', 'job_id': job_id,
csrf_field: csrf_token})
output_objects.append({'object_type': 'html_form', 'text': helper})
job_obj['cancellink'] = {'object_type': 'link',
'destination':
"javascript: %s();" % js_name,
'text': 'Cancel job'}
target_op = 'jobschedule'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
js_name = 'jobschedule%s' % hexlify(job_id)
helper = html_post_helper(js_name, '%s.py' % target_op,
{'job_id': job_id,
csrf_field: csrf_token})
output_objects.append({'object_type': 'html_form', 'text': helper})
job_obj['jobschedulelink'] = {'object_type': 'link',
'destination':
"javascript: %s();" % js_name,
'text':
'Request schedule information'}
target_op = 'jobfeasible'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
js_name = 'jobfeasible%s' % hexlify(job_id)
helper = html_post_helper(js_name, '%s.py' % target_op,
{'job_id': job_id,
csrf_field: csrf_token})
output_objects.append({'object_type': 'html_form', 'text': helper})
job_obj['jobfeasiblelink'] = {'object_type': 'link',
'destination':
"javascript: %s();" % js_name,
'text': 'Check job feasibility'}
job_obj['liveiolink'] = {'object_type': 'link',
'destination': 'liveio.py?job_id=%s' %
job_id, 'text': 'Request live I/O'}
job_list['jobs'].append(job_obj)
output_objects.append(job_list)
return (output_objects, status)
def main(client_id, user_arguments_dict, environ=None):
"""Main function used by front end"""
if environ is None:
environ = os.environ
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
status = returnvalues.OK
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path can use wildcards, dst cannot
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
dst = accepted['dst'][-1].lstrip(os.sep)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
if dst:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
dst_mode = "wb"
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
relative_dst = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
logger.warning('%s tried to %s into restricted path %s ! (%s)' %
(client_id, op_name, abs_dest, dst))
output_objects.append({
'object_type': 'error_text',
'text': "invalid destination: '%s'" % dst
})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
'accessed', [relative_path])
fd = open(abs_path, 'r')
# use file directly as iterator for efficiency
for line in fd:
output_lines.append(line)
fd.close()
except Exception as exc:
if not isinstance(exc, GDPIOLogError):
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'accessed', [relative_path],
failed=True,
details=exc)
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if dst:
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
'modified', [dst])
out_fd = open(abs_dest, dst_mode)
out_fd.writelines(output_lines)
out_fd.close()
logger.info('%s %s %s done' %
(op_name, abs_path, abs_dest))
except Exception as exc:
if not isinstance(exc, GDPIOLogError):
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'modified', [dst],
failed=True,
details=exc)
output_objects.append({
'object_type': 'error_text',
'text': "write failed: '%s'" % exc
})
logger.error("%s: write failed on '%s': %s" %
(op_name, abs_dest, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({
'object_type':
'text',
'text':
"wrote %s to %s" % (relative_path, relative_dst)
})
# Prevent truncate after first write
dst_mode = "ab+"
else:
entry = {
'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']
}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
# TODO: rip this hack out into real download handler?
# Force download of files when output_format == 'file_format'
# This will only work for the first file matching a glob when
# using file_format.
# And it is supposed to only work for one file.
if 'output_format' in user_arguments_dict:
output_format = user_arguments_dict['output_format'][0]
if output_format == 'file':
output_objects.append({
'object_type':
'start',
'headers': [('Content-Disposition',
'attachment; filename="%s";' %
os.path.basename(abs_path))]
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Delete frozen archive'
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path cannot use wildcards here
typecheck_overrides={},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flavor = accepted['flavor'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append(
{'object_type': 'error_text', 'text': '''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not flavor in freeze_flavors.keys():
output_objects.append({'object_type': 'error_text', 'text':
'Invalid freeze flavor: %s' % flavor})
return (output_objects, returnvalues.CLIENT_ERROR)
title = freeze_flavors[flavor]['deletefreeze_title']
output_objects.append({'object_type': 'header', 'text': title})
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = title
if not configuration.site_enable_freeze:
output_objects.append({'object_type': 'text', 'text':
'''Freezing archives is disabled on this site.
Please contact the site admins %s if you think it should be enabled.
''' % configuration.admin_email})
return (output_objects, returnvalues.OK)
freeze_id = accepted['freeze_id'][-1]
target = accepted['target'][-1]
path_list = accepted['path']
if not target in valid_targets:
output_objects.append({'object_type': 'error_text', 'text':
'Invalid delete freeze target: %s' % target})
return (output_objects, returnvalues.CLIENT_ERROR)
# NB: the restrictions on freeze_id prevents illegal directory traversal
if not is_frozen_archive(client_id, freeze_id, configuration):
logger.error("%s: invalid freeze '%s': %s" % (op_name,
client_id, freeze_id))
output_objects.append({'object_type': 'error_text',
'text': "No such frozen archive: '%s'"
% freeze_id})
return (output_objects, returnvalues.CLIENT_ERROR)
(load_status, freeze_dict) = get_frozen_archive(client_id, freeze_id,
configuration,
checksum_list=[])
if not load_status:
logger.error("%s: load failed for '%s': %s" %
(op_name, freeze_id, freeze_dict))
output_objects.append(
{'object_type': 'error_text',
'text': 'Could not read frozen archive details for %s'
% freeze_id})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Make sure the frozen archive belongs to the user trying to delete it
if client_id != freeze_dict['CREATOR']:
logger.error("%s: illegal access attempt for '%s': %s" %
(op_name, freeze_id, client_id))
output_objects.append({'object_type': 'error_text', 'text':
'You are not the owner of frozen archive "%s"'
% freeze_id})
return (output_objects, returnvalues.CLIENT_ERROR)
if freeze_dict.get('FLAVOR', 'freeze') != flavor:
logger.error("%s: flavor mismatch for '%s': %s vs %s" %
(op_name, freeze_id, flavor, freeze_dict))
output_objects.append({'object_type': 'error_text', 'text':
'No such %s archive "%s"' % (flavor, freeze_id)})
return (output_objects, returnvalues.CLIENT_ERROR)
# Prevent user-delete of the frozen archive if configuration forbids it.
# We exclude any archives in the pending intermediate freeze state.
# Freeze admins are also excluded from the restrictions.
state = freeze_dict.get('STATE', keyword_final)
if state == keyword_updating:
output_objects.append(
{'object_type': 'error_text', 'text':
"Can't change %s archive %s which is currently being updated" %
(flavor, freeze_id)})
output_objects.append({
'object_type': 'link',
'destination': 'showfreeze.py?freeze_id=%s;flavor=%s' %
(freeze_id, flavor),
'class': 'viewarchivelink iconspace genericbutton',
'title': 'View details about your %s archive' % flavor,
'text': 'View details',
})
return (output_objects, returnvalues.CLIENT_ERROR)
elif state == keyword_final and \
flavor in configuration.site_permanent_freeze and \
not client_id in configuration.site_freeze_admins:
output_objects.append(
{'object_type': 'error_text', 'text':
"Can't change %s archives like '%s' yourself due to site policy"
% (flavor, freeze_id)})
return (output_objects, returnvalues.CLIENT_ERROR)
client_dir = client_id_dir(client_id)
user_archives = os.path.join(configuration.freeze_home, client_dir)
# Please note that base_dir must end in slash to avoid access to other
# user archive dirs if own name is a prefix of another archive name
base_dir = os.path.abspath(os.path.join(user_archives, freeze_id)) + os.sep
if target == TARGET_ARCHIVE:
# Delete the entire freeze archive
(del_status, msg) = delete_frozen_archive(freeze_dict, client_id,
configuration)
# If something goes wrong when trying to delete freeze archive
# freeze_id, an error is displayed.
if not del_status:
logger.error("%s: failed for '%s': %s" % (op_name,
freeze_id, msg))
output_objects.append(
{'object_type': 'error_text', 'text':
'Could not remove entire %s archive %s: %s' %
(flavor, freeze_id, msg)})
return (output_objects, returnvalues.SYSTEM_ERROR)
# If deletion of frozen archive freeze_id is successful, we just
# return OK
else:
logger.info("%s: successful for '%s': %s" % (op_name,
freeze_id, client_id))
output_objects.append(
{'object_type': 'text', 'text':
'Successfully deleted %s archive: "%s"' % (flavor,
freeze_id)})
elif target == TARGET_PATH:
# Delete individual files in non-final archive
del_paths = []
for path in path_list:
# IMPORTANT: path must be expanded to abs for proper chrooting
server_path = os.path.join(base_dir, path)
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, False):
# Out of bounds!
logger.warning('%s tried to %s del restricted path %s ! ( %s)'
% (client_id, op_name, abs_path, path))
output_objects.append(
{'object_type': 'error_text', 'text':
'Not allowed to delete %s - outside archive %s !' %
(path, freeze_id)})
continue
del_paths.append(path)
(del_status, msg_list) = delete_archive_files(freeze_dict, client_id,
del_paths, configuration)
# If something goes wrong when trying to delete files from archive
# freeze_id, an error is displayed.
if not del_status:
logger.error("%s: delete archive file(s) failed for '%s':\n%s" %
(op_name, freeze_id, '\n'.join(msg_list)))
output_objects.append(
{'object_type': 'error_text', 'text':
'Could not remove file(s) from archive %s: %s'
% (freeze_id, '\n '.join(msg_list))})
return (output_objects, returnvalues.SYSTEM_ERROR)
# If deletion of files from archive freeze_id is successful, we just
# return OK
else:
logger.info("%s: delete %d files successful for '%s': %s" %
(op_name, len(path_list), freeze_id, client_id))
output_objects.append(
{'object_type': 'text', 'text':
'Successfully deleted %d file(s) from archive: "%s"' %
(len(path_list), freeze_id)})
# Success - show link to overview
output_objects.append({'object_type': 'link', 'destination':
'freezedb.py',
'class': 'infolink iconspace',
'title': 'Show archives',
'text': 'Show archives'})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict, environ=None):
"""Main function used by front end"""
if environ is None:
environ = os.environ
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False,
op_menu=client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input(
user_arguments_dict,
defaults,
output_objects,
allow_rejects=False,
# NOTE: path can use wildcards
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
iosessionid = accepted['iosessionid'][-1]
share_id = accepted['share_id'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Either authenticated user client_id set or sharelink ID
if client_id:
user_id = client_id
target_dir = client_id_dir(client_id)
base_dir = configuration.user_home
id_query = ''
page_title = 'Remove User File'
if force(flags):
rm_helper = delete_path
else:
rm_helper = remove_path
userstyle = True
widgets = True
elif share_id:
try:
(share_mode, _) = extract_mode_id(configuration, share_id)
except ValueError as err:
logger.error('%s called with invalid share_id %s: %s' %
(op_name, share_id, err))
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid sharelink ID: %s' % share_id
})
return (output_objects, returnvalues.CLIENT_ERROR)
# TODO: load and check sharelink pickle (currently requires client_id)
user_id = 'anonymous user through share ID %s' % share_id
if share_mode == 'read-only':
logger.error('%s called without write access: %s' %
(op_name, accepted))
output_objects.append({
'object_type': 'error_text',
'text': 'No write access!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
target_dir = os.path.join(share_mode, share_id)
base_dir = configuration.sharelink_home
id_query = '?share_id=%s' % share_id
page_title = 'Remove Shared File'
rm_helper = delete_path
userstyle = False
widgets = False
elif iosessionid.strip() and iosessionid.isalnum():
user_id = iosessionid
base_dir = configuration.webserver_home
target_dir = iosessionid
page_title = 'Remove Session File'
rm_helper = delete_path
userstyle = False
widgets = False
else:
logger.error('%s called without proper auth: %s' % (op_name, accepted))
output_objects.append({
'object_type': 'error_text',
'text': 'Authentication is missing!'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(base_dir, target_dir)) + os.sep
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = page_title
title_entry['skipwidgets'] = not widgets
title_entry['skipuserstyle'] = not userstyle
output_objects.append({'object_type': 'header', 'text': page_title})
logger.debug("%s: with paths: %s" % (op_name, pattern_list))
# Input validation assures target_dir can't escape base_dir
if not os.path.isdir(base_dir):
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid client/sharelink/session id!'
})
logger.warning('%s used %s with invalid base dir: %s' %
(user_id, op_name, base_dir))
return (output_objects, returnvalues.CLIENT_ERROR)
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! ( %s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
logger.warning("%s: no matching paths: %s" %
(op_name, pattern_list))
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
real_path = os.path.realpath(abs_path)
relative_path = abs_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({
'object_type': 'file',
'name': relative_path
})
# Make it harder to accidentially delete too much - e.g. do not
# delete VGrid files without explicit selection of subdir contents
if abs_path == os.path.abspath(base_dir):
logger.error("%s: refusing rm home dir: %s" %
(op_name, abs_path))
output_objects.append({
'object_type':
'warning',
'text':
"You're not allowed to delete your entire home directory!"
})
status = returnvalues.CLIENT_ERROR
continue
# Generally refuse handling symlinks including root vgrid shares
elif os.path.islink(abs_path):
logger.error("%s: refusing rm link: %s" % (op_name, abs_path))
output_objects.append({
'object_type':
'warning',
'text':
"""
You're not allowed to delete entire special folders like %s shares and %s
""" % (configuration.site_vgrid_label, trash_linkname)
})
status = returnvalues.CLIENT_ERROR
continue
# Additionally refuse operations on inherited subvgrid share roots
elif in_vgrid_share(configuration, abs_path) == relative_path:
output_objects.append({
'object_type':
'warning',
'text':
"""You're not allowed to
remove entire %s shared folders!""" % configuration.site_vgrid_label
})
status = returnvalues.CLIENT_ERROR
continue
elif os.path.isdir(abs_path) and not recursive(flags):
logger.error("%s: non-recursive call on dir '%s'" %
(op_name, abs_path))
output_objects.append({
'object_type':
'error_text',
'text':
"cannot remove '%s': is a direcory" % relative_path
})
status = returnvalues.CLIENT_ERROR
continue
trash_base = get_trash_location(configuration, abs_path)
if not trash_base and not force(flags):
logger.error("%s: no trash for dir '%s'" % (op_name, abs_path))
output_objects.append({
'object_type':
'error_text',
'text':
"No trash enabled for '%s' - read-only?" % relative_path
})
status = returnvalues.CLIENT_ERROR
continue
try:
if rm_helper == remove_path and \
os.path.commonprefix([real_path, trash_base]) \
== trash_base:
logger.warning("%s: already in trash: '%s'" %
(op_name, real_path))
output_objects.append({
'object_type':
'error_text',
'text':
"""
'%s' is already in trash - no action: use force flag to permanently delete""" %
relative_path
})
status = returnvalues.CLIENT_ERROR
continue
except Exception as err:
logger.error("%s: check trash failed: %s" % (op_name, err))
continue
if not check_write_access(abs_path):
logger.warning('%s called without write access: %s' %
(op_name, abs_path))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot remove "%s": inside a read-only location!' %
pattern
})
status = returnvalues.CLIENT_ERROR
continue
# TODO: limit delete in vgrid share trash to vgrid owners / conf?
# ... malicious members can still e.g. truncate all files.
# we could consider removing write bit on move to trash.
# TODO: user setting to switch on/off trash?
# TODO: add direct delete checkbox in fileman move to trash dialog?
# TODO: add empty trash option for Trash?
# TODO: user settings to define read-only and auto-expire in trash?
# TODO: add trash support for sftp/ftps/webdavs?
gdp_iolog_action = 'deleted'
gdp_iolog_paths = [relative_path]
if rm_helper == remove_path:
gdp_iolog_action = 'moved'
trash_base_path = \
get_trash_location(configuration, abs_path, True)
trash_relative_path = \
trash_base_path.replace(configuration.user_home, '')
trash_relative_path = \
trash_relative_path.replace(
configuration.vgrid_files_home, '')
gdp_iolog_paths.append(trash_relative_path)
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
gdp_iolog_action, gdp_iolog_paths)
gdp_iolog_status = True
except GDPIOLogError as exc:
gdp_iolog_status = False
rm_err = [str(exc)]
rm_status = False
if gdp_iolog_status:
(rm_status, rm_err) = rm_helper(configuration, abs_path)
if not rm_status or not gdp_iolog_status:
if gdp_iolog_status:
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
gdp_iolog_action,
gdp_iolog_paths,
failed=True,
details=rm_err)
logger.error("%s: failed on '%s': %s" %
(op_name, abs_path, ', '.join(rm_err)))
output_objects.append({
'object_type':
'error_text',
'text':
"remove '%s' failed: %s" %
(relative_path, '. '.join(rm_err))
})
status = returnvalues.SYSTEM_ERROR
continue
logger.info("%s: successfully (re)moved %s" % (op_name, abs_path))
output_objects.append({
'object_type': 'text',
'text': "removed %s" % (relative_path)
})
output_objects.append({
'object_type': 'link',
'destination': 'ls.py%s' % id_query,
'text': 'Return to files overview'
})
return (output_objects, status)
def main(client_id, user_arguments_dict, environ=None):
"""Main function used by front end"""
if environ is None:
environ = os.environ
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False,
op_menu=client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input(
user_arguments_dict,
defaults,
output_objects,
allow_rejects=False,
# NOTE: path can use wildcards, current_dir cannot
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
current_dir = accepted['current_dir'][-1].lstrip('/')
share_id = accepted['share_id'][-1]
status = returnvalues.OK
read_mode, write_mode = True, True
# Either authenticated user client_id set or sharelink ID
if client_id:
user_id = client_id
target_dir = client_id_dir(client_id)
base_dir = configuration.user_home
redirect_name = configuration.site_user_redirect
redirect_path = redirect_name
id_args = ''
root_link_name = 'USER HOME'
main_class = "user_ls"
page_title = 'User Files'
userstyle = True
widgets = True
visibility_mods = '''
.%(main_class)s .disable_read { display: none; }
.%(main_class)s .disable_write { display: none; }
'''
elif share_id:
try:
(share_mode, _) = extract_mode_id(configuration, share_id)
except ValueError as err:
logger.error('%s called with invalid share_id %s: %s' %
(op_name, share_id, err))
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid sharelink ID: %s' % share_id
})
return (output_objects, returnvalues.CLIENT_ERROR)
# TODO: load and check sharelink pickle (currently requires client_id)
# then include shared by %(owner)s on page header
user_id = 'anonymous user through share ID %s' % share_id
target_dir = os.path.join(share_mode, share_id)
base_dir = configuration.sharelink_home
redirect_name = 'share_redirect'
redirect_path = os.path.join(redirect_name, share_id)
id_args = 'share_id=%s;' % share_id
root_link_name = '%s' % share_id
main_class = "sharelink_ls"
page_title = 'Shared Files'
userstyle = False
widgets = False
# default to include file info
if flags == '':
flags += 'f'
if share_mode == 'read-only':
write_mode = False
visibility_mods = '''
.%(main_class)s .enable_write { display: none; }
.%(main_class)s .disable_read { display: none; }
'''
elif share_mode == 'write-only':
read_mode = False
visibility_mods = '''
.%(main_class)s .enable_read { display: none; }
.%(main_class)s .disable_write { display: none; }
'''
else:
visibility_mods = '''
.%(main_class)s .disable_read { display: none; }
.%(main_class)s .disable_write { display: none; }
'''
else:
logger.error('%s called without proper auth: %s' % (op_name, accepted))
output_objects.append({
'object_type': 'error_text',
'text': 'Authentication is missing!'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
visibility_toggle = '''
<style>
%s
</style>
''' % (visibility_mods % {
'main_class': main_class
})
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(base_dir, target_dir)) + os.sep
if not os.path.isdir(base_dir):
logger.error('%s called on missing base_dir: %s' % (op_name, base_dir))
output_objects.append({
'object_type': 'error_text',
'text': 'No such %s!' % page_title.lower()
})
return (output_objects, returnvalues.CLIENT_ERROR)
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = page_title
title_entry['skipwidgets'] = not widgets
title_entry['skipuserstyle'] = not userstyle
user_settings = title_entry.get('user_settings', {})
open_button_id = 'open_fancy_upload'
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
fill_helpers = {
'dest_dir': current_dir + os.sep,
'share_id': share_id,
'flags': flags,
'tmp_flags': flags,
'long_set': long_list(flags),
'recursive_set': recursive(flags),
'all_set': all(flags),
'fancy_open': open_button_id,
'fancy_dialog': fancy_upload_html(configuration),
'form_method': form_method,
'csrf_field': csrf_field,
'csrf_limit': csrf_limit
}
target_op = 'uploadchunked'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token})
(cf_import, cf_init, cf_ready) = confirm_js(configuration)
(fu_import, fu_init,
fu_ready) = fancy_upload_js(configuration,
'function() { location.reload(); }', share_id,
csrf_token)
add_import = '''
%s
%s
''' % (cf_import, fu_import)
add_init = '''
%s
%s
%s
%s
''' % (cf_init, fu_init, select_all_javascript(),
selected_file_actions_javascript())
add_ready = '''
%s
%s
/* wrap openFancyUpload in function to avoid event data as argument */
$("#%s").click(function() { openFancyUpload(); });
$("#checkall_box").click(toggleChecked);
''' % (cf_ready, fu_ready, open_button_id)
# TODO: can we update style inline to avoid explicit themed_styles?
styles = themed_styles(
configuration,
advanced=['jquery.fileupload.css', 'jquery.fileupload-ui.css'],
skin=['fileupload-ui.custom.css'],
user_settings=user_settings)
styles['advanced'] += '''
%s
''' % visibility_toggle
title_entry['style'] = styles
title_entry['script']['advanced'] += add_import
title_entry['script']['init'] += add_init
title_entry['script']['ready'] += add_ready
title_entry['script']['body'] = ' class="%s"' % main_class
output_objects.append({'object_type': 'header', 'text': page_title})
# TODO: move to output html handler
output_objects.append({
'object_type': 'html_form',
'text': confirm_html(configuration)
})
# Shared URL helpers
ls_url_template = 'ls.py?%scurrent_dir=%%(rel_dir_enc)s;flags=%s' % \
(id_args, flags)
csrf_token = make_csrf_token(configuration, form_method, 'rm', client_id,
csrf_limit)
rm_url_template = 'rm.py?%spath=%%(rel_path_enc)s;%s=%s' % \
(id_args, csrf_field, csrf_token)
rmdir_url_template = 'rm.py?%spath=%%(rel_path_enc)s;flags=r;%s=%s' % \
(id_args, csrf_field, csrf_token)
editor_url_template = 'editor.py?%spath=%%(rel_path_enc)s' % id_args
redirect_url_template = '/%s/%%(rel_path_enc)s' % redirect_path
location_pre_html = """
<div class='files'>
<table class='files'>
<tr class=title><td class=centertext>
Working directory:
</td></tr>
<tr><td class='centertext'>
"""
output_objects.append({
'object_type': 'html_form',
'text': location_pre_html
})
# Use current_dir nav location links
for pattern in pattern_list[:1]:
links = []
links.append({
'object_type': 'link',
'text': root_link_name,
'destination': ls_url_template % {
'rel_dir_enc': '.'
}
})
prefix = ''
parts = os.path.normpath(current_dir).split(os.sep)
for i in parts:
if i == ".":
continue
prefix = os.path.join(prefix, i)
links.append({
'object_type': 'link',
'text': i,
'destination': ls_url_template % {
'rel_dir_enc': quote(prefix)
}
})
output_objects.append({
'object_type': 'multilinkline',
'links': links,
'sep': ' %s ' % os.sep
})
location_post_html = """
</td></tr>
</table>
</div>
<br />
"""
output_objects.append({
'object_type': 'html_form',
'text': location_post_html
})
more_html = """
<div class='files if_full'>
<form method='%(form_method)s' name='fileform' onSubmit='return selectedFilesAction();'>
<table class='files'>
<tr class=title><td class=centertext colspan=2>
Advanced file actions
</td></tr>
<tr><td>
Action on paths selected below
(please hold mouse cursor over button for a description):
</td>
<td class=centertext>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='v' />
<input type='submit' title='Show concatenated contents (cat)' onClick='document.pressed=this.value' value='cat' />
<input type='submit' onClick='document.pressed=this.value' value='head' title='Show first lines (head)' />
<input type='submit' onClick='document.pressed=this.value' value='tail' title='Show last lines (tail)' />
<input type='submit' onClick='document.pressed=this.value' value='wc' title='Count lines/words/chars (wc)' />
<input type='submit' onClick='document.pressed=this.value' value='stat' title='Show details (stat)' />
<input type='submit' onClick='document.pressed=this.value' value='touch' title='Update timestamp (touch)' />
<input type='submit' onClick='document.pressed=this.value' value='truncate' title='truncate! (truncate)' />
<input type='submit' onClick='document.pressed=this.value' value='rm' title='delete! (rm)' />
<input type='submit' onClick='document.pressed=this.value' value='rmdir' title='Remove directory (rmdir)' />
<input type='submit' onClick='document.pressed=this.value' value='submit' title='Submit file (submit)' />
</td></tr>
</table>
</form>
</div>
""" % {
'form_method': form_method
}
output_objects.append({'object_type': 'html_form', 'text': more_html})
dir_listings = []
output_objects.append({
'object_type': 'dir_listings',
'dir_listings': dir_listings,
'flags': flags,
'redirect_name': redirect_name,
'redirect_path': redirect_path,
'share_id': share_id,
'ls_url_template': ls_url_template,
'rm_url_template': rm_url_template,
'rmdir_url_template': rmdir_url_template,
'editor_url_template': editor_url_template,
'redirect_url_template': redirect_url_template,
})
first_match = None
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
current_path = os.path.normpath(os.path.join(base_dir, current_dir))
unfiltered_match = glob.glob(current_path + os.sep + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(user_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
if not first_match:
first_match = abs_path
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
# Never show any ls output in write-only mode (css hide is not enough!)
if not read_mode:
continue
for abs_path in match:
if abs_path + os.sep == base_dir:
relative_path = '.'
else:
relative_path = abs_path.replace(base_dir, '')
entries = []
dir_listing = {
'object_type': 'dir_listing',
'relative_path': relative_path,
'entries': entries,
'flags': flags,
}
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
'accessed', [relative_path])
except GDPIOLogError as exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
continue
handle_ls(configuration, output_objects, entries, base_dir,
abs_path, flags, 0)
dir_listings.append(dir_listing)
output_objects.append({
'object_type':
'html_form',
'text':
"""<br/>
<div class='files disable_read'>
<p class='info icon'>"""
})
# Shared message for text (e.g. user scripts) and html-format
if not read_mode:
# Please note that we use verbatim to get info icon right in html
output_objects.append({
'object_type':
'verbatim',
'text':
"""
This is a write-only share so you do not have access to see the files, only
upload data and create directories.
"""
})
output_objects.append({
'object_type':
'html_form',
'text':
"""
</p>
</div>
<div class='files enable_read'>
<form method='get' action='ls.py'>
<table class='files'>
<tr class=title><td class=centertext>
Filter paths (wildcards like * and ? are allowed)
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
<input type='text' name='path' value='' />
<input type='submit' value='Filter' />
</td></tr>
</table>
</form>
</div>
""" % fill_helpers
})
# Short/long format buttons
fill_helpers['tmp_flags'] = flags + 'l'
htmlform = """
<table class='files if_full'>
<tr class=title><td class=centertext colspan=4>
File view options
</td></tr>
<tr><td colspan=4><br /></td></tr>
<tr class=title><td>Parameter</td><td>Setting</td><td>Enable</td><td>Disable</td></tr>
<tr><td>Long format</td><td>
%(long_set)s</td><td>
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />" % entry
fill_helpers['tmp_flags'] = flags.replace('l', '')
htmlform += """
<input type='submit' value='On' /><br />
</form>
</td><td>
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />" % entry
htmlform += """
<input type='submit' value='Off' /><br />
</form>
</td></tr>
"""
# Recursive output
fill_helpers['tmp_flags'] = flags + 'r'
htmlform += """
<!-- Non-/recursive list buttons -->
<tr><td>Recursion</td><td>
%(recursive_set)s</td><td>""" % fill_helpers
htmlform += """
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />" % entry
fill_helpers['tmp_flags'] = flags.replace('r', '')
htmlform += """
<input type='submit' value='On' /><br />
</form>
</td><td>
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />"\
% entry
htmlform += """
<input type='submit' value='Off' /><br />
</form>
</td></tr>
"""
htmlform += """
<!-- Show dot files buttons -->
<tr><td>Show hidden files</td><td>
%(all_set)s</td><td>""" % fill_helpers
fill_helpers['tmp_flags'] = flags + 'a'
htmlform += """
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />" % entry
fill_helpers['tmp_flags'] = flags.replace('a', '')
htmlform += """
<input type='submit' value='On' /><br />
</form>
</td><td>
<form method='get' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%(tmp_flags)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
""" % fill_helpers
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />" % entry
htmlform += """
<input type='submit' value='Off' /><br />
</form>
</td></tr>
</table>
"""
# show flag buttons after contents to limit clutter
output_objects.append({'object_type': 'html_form', 'text': htmlform})
# create additional action forms
if first_match:
htmlform = """
<br />
<div class='files disable_write'>
<p class='info icon'>
This is a read-only share so you do not have access to edit or add files, only
view data.
</p>
</div>
<table class='files enable_write if_full'>
<tr class=title><td class=centertext colspan=3>
Edit file
</td></tr>
<tr><td>
Fill in the path of a file to edit and press 'edit' to open that file in the<br />
online file editor. Alternatively a file can be selected for editing through<br />
the listing of personal files.
</td><td colspan=2 class=righttext>
<form name='editor' method='get' action='editor.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
<input type='text' name='path' size=50 value='' required />
<input type='submit' value='edit' />
</form>
</td></tr>
</table>
<br />""" % fill_helpers
target_op = 'mkdir'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token})
htmlform += """
<table class='files enable_write'>
<tr class=title><td class=centertext colspan=4>
Create directory
</td></tr>
<tr><td>
Name of new directory to be created in current directory (%(dest_dir)s)
</td><td class=righttext colspan=3>
<form method='%(form_method)s' action='%(target_op)s.py'>
<input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
<input name='path' size=50 required />
<input type='submit' value='Create' name='mkdirbutton' />
</form>
</td></tr>
</table>
<br />
""" % fill_helpers
target_op = 'textarea'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token})
htmlform += """
<form enctype='multipart/form-data' method='%(form_method)s'
action='%(target_op)s.py'>
<input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' />
<input type='hidden' name='share_id' value='%(share_id)s' />
<table class='files enable_write if_full'>
<tr class='title'><td class=centertext colspan=4>
Upload file
</td></tr>
<tr><td colspan=4>
Upload file to current directory (%(dest_dir)s)
</td></tr>
<tr class='if_full'><td colspan=2>
Extract package files (.zip, .tar.gz, .tar.bz2)
</td><td colspan=2>
<input type=checkbox name='extract_0' />
</td></tr>
<tr class='if_full'><td colspan=2>
Submit mRSL files (also .mRSL files included in packages)
</td><td colspan=2>
<input type=checkbox name='submitmrsl_0' />
</td></tr>
<tr><td>
File to upload
</td><td class=righttext colspan=3>
<input name='fileupload_0_0_0' type='file'/>
</td></tr>
<tr><td>
Optional remote filename (extra useful in windows)
</td><td class=righttext colspan=3>
<input name='default_remotefilename_0' type='hidden' value='%(dest_dir)s'/>
<input name='remotefilename_0' type='text' size='50' value='%(dest_dir)s'/>
<input type='submit' value='Upload' name='sendfile'/>
</td></tr>
</table>
</form>
%(fancy_dialog)s
<table class='files enable_write'>
<tr class='title'><td class='centertext'>
Upload files efficiently (using chunking).
</td></tr>
<tr><td class='centertext'>
<button id='%(fancy_open)s'>Open Upload dialog</button>
</td></tr>
</table>
<script type='text/javascript' >
setUploadDest('%(dest_dir)s');
</script>
""" % fill_helpers
output_objects.append({'object_type': 'html_form', 'text': htmlform})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
patterns = accepted['job_id']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
status = returnvalues.OK
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to
# avoid leaking information about file system layout while
# allowing consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match^I
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % pattern
})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
# job feasibility is hard on the server, limit
if len(filelist) > 100:
output_objects.append({
'object_type':
'error_text',
'text':
'Too many matching jobs (%s)!' % len(filelist)
})
return (output_objects, returnvalues.CLIENT_ERROR)
checkcondjobs = []
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
checkcondjob = {'object_type': 'checkcondjob', 'job_id': job_id}
dict = unpickle(filepath, logger)
if not dict:
checkcondjob['message'] = \
('The file containing the information ' \
'for job id %s could not be opened! ' \
'You can only check feasibility of ' \
'your own jobs!'
) % job_id
checkcondjobs.append(checkcondjob)
status = returnvalues.CLIENT_ERROR
continue
# Is the job status pending?
possible_check_states = ['QUEUED', 'RETRY', 'FROZEN']
if not dict['STATUS'] in possible_check_states:
checkcondjob['message'] = \
'You can only check feasibility of jobs with status: %s.'\
% ' or '.join(possible_check_states)
checkcondjobs.append(checkcondjob)
continue
# Actually check feasibility
feasible_res = job_feasibility(configuration, dict)
checkcondjob.update(feasible_res)
checkcondjobs.append(checkcondjob)
output_objects.append({
'object_type': 'checkcondjobs',
'checkcondjobs': checkcondjobs
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False,
op_menu=client_id)
# NOTE: we expose editor e.g. on sharelinks (ls) but do not allow it, yet
if not client_id:
output_objects.append({
'object_type':
'error_text',
'text':
'The inline editor is currently only available for authenticated '
'users on this site'
})
return (output_objects, returnvalues.CLIENT_ERROR)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path cannot use wildcards here
typecheck_overrides={},
)
# TODO: if validator is too tight we should accept rejects here
# and then make sure that such rejected fields are never printed
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
path = accepted['path'][-1]
current_dir = accepted['current_dir'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
# the client can choose to specify the path of the target directory with
# current_dir + "/" + path, instead of specifying the complete path in
# subdirs. This is usefull from ls.py where a hidden html control makes it
# possible to target the directory from the current dir.
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = '%s file web editor' % configuration.short_title
# TODO: fully update to new style and script structure
title_entry['style']['advanced'] += advanced_editor_css_deps()
title_entry['script']['advanced'] += advanced_editor_js_deps() + '\n'
title_entry['script']['advanced'] += lock_info('this file', -1)
output_objects.append({
'object_type':
'header',
'text':
'Editing file in %s home directory' % configuration.short_title
})
if not path:
now = time.gmtime()
path = 'noname-%s.txt' % time.strftime('%d%m%y-%H%M%S', now)
output_objects.append({
'object_type':
'text',
'text':
'No path supplied - creating new file in %s' % path
})
rel_path = os.path.join(current_dir.lstrip(os.sep), path.lstrip(os.sep))
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, rel_path))
if not valid_user_path(configuration, abs_path, base_dir):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, rel_path))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
(owner, time_left) = acquire_edit_lock(abs_path, client_id)
if owner == client_id:
javascript = \
'''<script type="text/javascript">
setTimeout("newcountdown('%s', %d)", 1)
</script>
'''\
% (path, time_left / 60)
output_objects.append({'object_type': 'html_form', 'text': javascript})
html = edit_file(configuration, client_id, path, abs_path)
output_objects.append({'object_type': 'html_form', 'text': html})
else:
output_objects.append({
'object_type':
'error_text',
'text':
'%s acquired the editing lock for %s! (timeout in %d seconds)' %
(owner, path, time_left)
})
return (output_objects, returnvalues.CLIENT_ERROR)
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path can use wildcards, dst and current_dir cannot
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
algo_list = accepted['hash_algo']
max_chunks = int(accepted['max_chunks'][-1])
pattern_list = accepted['path']
dst = accepted['dst'][-1]
current_dir = accepted['current_dir'][-1].lstrip(os.sep)
# All paths are relative to current_dir
pattern_list = [os.path.join(current_dir, i) for i in pattern_list]
if dst:
dst = os.path.join(current_dir, dst)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dir = os.path.abspath(
os.path.join(base_dir, current_dir.lstrip(os.sep)))
if not valid_user_path(configuration, abs_dir, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"You're not allowed to work in %s!" % current_dir
})
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_dir, current_dir))
return (output_objects, returnvalues.CLIENT_ERROR)
if verbose(flags):
output_objects.append({
'object_type': 'text',
'text': "working in %s" % current_dir
})
if dst:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# NOTE: dst already incorporates current_dir prefix here
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
logger.info('chksum in %s' % abs_dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % dst
})
logger.warning('%s tried to %s restricted path %s !(%s)' %
(client_id, op_name, abs_dest, dst))
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_dest))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot checksum to "%s": inside a read-only location!' %
relative_dest
})
return (output_objects, returnvalues.CLIENT_ERROR)
all_lines = []
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
output_lines = []
for hash_algo in algo_list:
try:
chksum_helper = _algo_map.get(hash_algo, _algo_map["md5"])
checksum = chksum_helper(abs_path, max_chunks=max_chunks)
line = "%s %s\n" % (checksum, relative_path)
logger.info("%s %s of %s: %s" %
(op_name, hash_algo, abs_path, checksum))
output_lines.append(line)
except Exception as exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output', 'lines': output_lines}
output_objects.append(entry)
all_lines += output_lines
if dst and not write_file(''.join(all_lines), abs_dest, logger):
output_objects.append({
'object_type':
'error_text',
'text':
"failed to write checksums to %s" % relative_dest
})
logger.error("writing checksums to %s for %s failed" %
(abs_dest, client_id))
status = returnvalues.SYSTEM_ERROR
return (output_objects, status)
def main(client_id, user_arguments_dict, environ=None):
"""Main function used by front end"""
if environ is None:
environ = os.environ
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False,
op_menu=client_id)
# TODO: this handler is incomplete and NOT yet hooked up with Xgi-bin
return (output_objects, returnvalues.SYSTEM_ERROR)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input(
user_arguments_dict,
defaults,
output_objects,
allow_rejects=False,
# NOTE: path cannot use wildcards here
typecheck_overrides={},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
filename = accepted['filename'][-1]
patterns = accepted['path']
iosessionid = accepted['iosessionid'][-1]
file_startpos = accepted['file_startpos'][-1]
file_endpos = accepted['file_endpos'][-1]
if file_startpos:
file_startpos = int(file_startpos)
else:
file_startpos = -1
if file_endpos:
file_endpos = int(file_endpos)
else:
file_endpos = -1
# Legacy naming
if filename:
patterns = [filename]
valid_methods = ['GET', 'PUT', 'DELETE']
action = os.getenv('REQUEST_METHOD')
if not action in valid_methods:
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting %s requests''' % ', '.join(valid_methods)
})
return (output_objects, returnvalues.CLIENT_ERROR)
# TODO: introduce CSRF protection here when clients support it
# if not safe_handler(configuration, action, op_name, client_id,
# get_csrf_limit(configuration), accepted):
# output_objects.append(
# {'object_type': 'error_text', 'text':
# '''Only accepting CSRF-filtered POST requests to prevent unintended updates'''
# })
# return (output_objects, returnvalues.CLIENT_ERROR)
# Either authenticated user client_id set or job IO session ID
if client_id:
user_id = client_id
target_dir = client_id_dir(client_id)
base_dir = configuration.user_home
page_title = 'User range file access'
widgets = True
userstyle = True
elif iosessionid:
user_id = iosessionid
target_dir = iosessionid
base_dir = configuration.webserver_home
page_title = 'Create Shared Directory'
widgets = False
userstyle = False
else:
logger.error('%s called without proper auth: %s' % (op_name, accepted))
output_objects.append({
'object_type': 'error_text',
'text': 'Authentication is missing!'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(base_dir, target_dir)) + os.sep
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = page_title
title_entry['skipwidgets'] = not widgets
title_entry['skipuserstyle'] = not userstyle
output_objects.append({'object_type': 'header', 'text': page_title})
# Input validation assures target_dir can't escape base_dir
if not os.path.isdir(base_dir):
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid client/iosession id!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
# NB: Globbing disabled on purpose here
unfiltered_match = [base_dir + os.sep + pattern]
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warn('%s tried to %s %s restricted path! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: cannot access file '%s': Permission denied" %
(op_name, pattern)
})
status = returnvalues.CLIENT_ERROR
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
if action == 'GET':
if not do_get(configuration, output_objects, abs_path,
file_startpos, file_endpos):
output_objects.append({
'object_type': 'error_text',
'text': '''Could not gett %r''' % pattern
})
status = returnvalues.SYSTEM_ERROR
elif action == 'PUT':
if not do_put(configuration, output_objects, abs_path,
file_startpos, file_endpos):
output_objects.append({
'object_type': 'error_text',
'text': '''Could not put %r''' % pattern
})
status = returnvalues.SYSTEM_ERROR
elif action == 'DELETE':
if not do_delete(configuration, output_objects, abs_path):
output_objects.append({
'object_type':
'error_text',
'text':
'''Could not delete %r''' % pattern
})
status = returnvalues.SYSTEM_ERROR
else:
output_objects.append({
'object_type': 'error_text',
'text': 'Unsupported action: %r' % action
})
return (output_objects, returnvalues.CLIENT_ERROR)
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
patterns = accepted['job_id']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not patterns:
output_objects.append({
'object_type': 'error_text',
'text': 'No job_id specified!'
})
return (output_objects, returnvalues.NO_SUCH_JOB_ID)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
filelist = []
keywords_dict = mrslkeywords.get_keywords_dict(configuration)
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % pattern
})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
# resubmit is hard on the server
if len(filelist) > 100:
output_objects.append({
'object_type':
'error_text',
'text':
'Too many matching jobs (%s)!' % len(filelist)
})
return (output_objects, returnvalues.CLIENT_ERROR)
resubmitobjs = []
status = returnvalues.OK
for filepath in filelist:
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
# ("Resubmitting job with job_id: %s" % job_id)
resubmitobj = {'object_type': 'resubmitobj', 'job_id': job_id}
mrsl_dict = unpickle(filepath, logger)
if not mrsl_dict:
resubmitobj['message'] = "No such job: %s (%s)" % (job_id,
mrsl_file)
status = returnvalues.CLIENT_ERROR
resubmitobjs.append(resubmitobj)
continue
resubmit_items = keywords_dict.keys()
# loop selected keywords and create mRSL string
resubmit_job_string = ''
for dict_elem in resubmit_items:
value = ''
# Extract job value with fallback to default to support optional
# fields
job_value = mrsl_dict.get(dict_elem,
keywords_dict[dict_elem]['Value'])
if keywords_dict[dict_elem]['Type'].startswith(
'multiplekeyvalues'):
for (elem_key, elem_val) in job_value:
if elem_key:
value += '%s=%s\n' % (str(elem_key).strip(),
str(elem_val).strip())
elif keywords_dict[dict_elem]['Type'].startswith('multiple'):
for elem in job_value:
if elem:
value += '%s\n' % str(elem).rstrip()
else:
if str(job_value):
value += '%s\n' % str(job_value).rstrip()
# Only insert keywords with an associated value
if value:
if value.rstrip() != '':
resubmit_job_string += '''::%s::
%s
''' % (dict_elem, value.rstrip())
# save tempfile
(filehandle, tempfilename) = \
tempfile.mkstemp(dir=configuration.mig_system_files,
text=True)
os.write(filehandle, resubmit_job_string)
os.close(filehandle)
# submit job the usual way
(new_job_status, msg, new_job_id) = new_job(tempfilename, client_id,
configuration, False, True)
if not new_job_status:
resubmitobj['status'] = False
resubmitobj['message'] = msg
status = returnvalues.SYSTEM_ERROR
resubmitobjs.append(resubmitobj)
continue
# o.out("Resubmit failed: %s" % msg)
# o.reply_and_exit(o.ERROR)
resubmitobj['status'] = True
resubmitobj['new_job_id'] = new_job_id
resubmitobjs.append(resubmitobj)
# o.out("Resubmit successful: %s" % msg)
# o.out("%s" % msg)
output_objects.append({
'object_type': 'resubmitobjs',
'resubmitobjs': resubmitobjs
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
title_entry = find_entry(output_objects, 'title')
label = "%s" % configuration.site_vgrid_label
title_entry['text'] = "Add/Update %s Trigger" % label
output_objects.append({
'object_type': 'header',
'text': 'Add/Update %s Trigger' % label
})
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
# NOTE: strip leftmost slashes from all fields used in file paths to avoid
# interference with os.path.join calls. Furthermore we strip and normalize
# the path variable first to make sure it does not point outside the vgrid.
# In practice any such directory traversal attempts will generally be moot
# since the grid_events daemon only starts a listener for each top-level
# vgrid and in there only reacts to events that match trigger rules from
# that particular vgrid. Thus only subvgrid access to parent vgrids might
# be a concern and still of limited consequence.
# NOTE: merge multi args into one string and split again to get flat array
rule_id = accepted['rule_id'][-1].strip()
vgrid_name = accepted['vgrid_name'][-1].strip().lstrip(os.sep)
path = os.path.normpath(accepted['path'][-1].strip()).lstrip(os.sep)
changes = [i.strip() for i in ' '.join(accepted['changes']).split()]
action = accepted['action'][-1].strip()
arguments = [
i.strip() for i in shlex.split(' '.join(accepted['arguments']))
]
rate_limit = accepted['rate_limit'][-1].strip()
settle_time = accepted['settle_time'][-1].strip()
match_files = accepted['match_files'][-1].strip() == 'True'
match_dirs = accepted['match_dirs'][-1].strip() == 'True'
match_recursive = accepted['match_recursive'][-1].strip() == 'True'
rank_str = accepted['rank'][-1]
try:
rank = int(rank_str)
except ValueError:
rank = None
logger.debug("addvgridtrigger with args: %s" % user_arguments_dict)
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
# we just use a high res timestamp as automatic rule_id
if rule_id == keyword_auto:
rule_id = "%d" % (time.time() * 1E8)
if action == keyword_auto:
action = valid_trigger_actions[0]
if any_state in changes:
changes = valid_trigger_changes
logger.info("addvgridtrigger %s" % vgrid_name)
# Validity of user and vgrid names is checked in this init function so
# no need to worry about illegal directory traversal through variables
(ret_val, msg, ret_variables) = \
init_vgrid_script_add_rem(vgrid_name, client_id,
rule_id, 'trigger',
configuration)
if not ret_val:
output_objects.append({'object_type': 'error_text', 'text': msg})
return (output_objects, returnvalues.CLIENT_ERROR)
elif msg:
# In case of warnings, msg is non-empty while ret_val remains True
output_objects.append({'object_type': 'warning', 'text': msg})
# if we get here user is either vgrid owner or allowed to add rule
# don't add if already in vgrid or parent vgrid - but update if owner
update_id = None
if vgrid_is_trigger(vgrid_name, rule_id, configuration):
if vgrid_is_trigger_owner(vgrid_name, rule_id, client_id,
configuration):
update_id = 'rule_id'
else:
output_objects.append({
'object_type':
'error_text',
'text':
'%s is already a trigger owned by somebody else in the %s' %
(rule_id, label)
})
return (output_objects, returnvalues.CLIENT_ERROR)
# don't add if already in subvgrid
(list_status, subvgrids) = vgrid_list_subvgrids(vgrid_name, configuration)
if not list_status:
output_objects.append({
'object_type':
'error_text',
'text':
'Error getting list of sub%ss: %s' % (label, subvgrids)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
for subvgrid in subvgrids:
if vgrid_is_trigger(subvgrid, rule_id, configuration, recursive=False):
output_objects.append({
'object_type': 'error_text',
'text': '''%(rule_id)s is already in a
sub-%(vgrid_label)s (%(subvgrid)s). Please remove the trigger from the
sub-%(vgrid_label)s and try again''' % {
'rule_id': rule_id,
'subvgrid': subvgrid,
'vgrid_label': label
}
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not action in valid_trigger_actions:
output_objects.append({
'object_type': 'error_text',
'text': "invalid action value %s" % action
})
return (output_objects, returnvalues.CLIENT_ERROR)
if keyword_all in changes:
changes = valid_trigger_changes
for change in changes:
if not change in valid_trigger_changes:
output_objects.append({
'object_type':
'error_text',
'text':
"found invalid change value %s" % change
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Check if we should load saved trigger for rank change or update
rule_dict = None
if rank is not None or update_id is not None:
(load_status, all_triggers) = vgrid_triggers(vgrid_name, configuration)
if not load_status:
output_objects.append({
'object_type':
'error_text',
'text':
'Failed to load triggers for %s: %s' %
(vgrid_name, all_triggers)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
for saved_dict in all_triggers:
if saved_dict['rule_id'] == rule_id:
rule_dict = saved_dict
break
if rule_dict is None:
output_objects.append({
'object_type':
'error_text',
'text':
'No such trigger %s for %s: %s' %
(rule_id, vgrid_name, all_triggers)
})
return (output_objects, returnvalues.CLIENT_ERROR)
elif not path:
# New trigger with missing path
output_objects.append({
'object_type': 'error_text',
'text': '''Either path or rank must
be set.'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
elif action == "submit" and not arguments:
# New submit trigger with missing mrsl arguments
output_objects.append({
'object_type':
'error_text',
'text':
'''Submit triggers must give
a job description file path as argument.'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Handle create and update (i.e. new, update all or just refresh mRSL)
if rank is None:
# IMPORTANT: we save the job template contents to avoid potential abuse
# Otherwise someone else in the VGrid could tamper with the template
# and make the next trigger execute arbitrary code on behalf of the
# rule owner.
templates = []
# Merge current and saved values
req_dict = {
'rule_id': rule_id,
'vgrid_name': vgrid_name,
'path': path,
'changes': changes,
'run_as': client_id,
'action': action,
'arguments': arguments,
'rate_limit': rate_limit,
'settle_time': settle_time,
'match_files': match_files,
'match_dirs': match_dirs,
'match_recursive': match_recursive,
'templates': templates
}
if rule_dict is None:
rule_dict = req_dict
else:
for field in user_arguments_dict:
if field in req_dict:
rule_dict[field] = req_dict[field]
# Now refresh template contents
if rule_dict['action'] == "submit":
for rel_path in rule_dict['arguments']:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, rel_path))
try:
if not valid_user_path(configuration, abs_path, base_dir,
True):
logger.warning(
'%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, rel_path))
raise ValueError('invalid submit path argument: %s' %
rel_path)
temp_fd = open(abs_path)
templates.append(temp_fd.read())
temp_fd.close()
except Exception as err:
logger.error("read submit argument file failed: %s" % err)
output_objects.append({
'object_type':
'error_text',
'text':
'failed to read submit argument file "%s"' % rel_path
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Save updated template contents here
rule_dict['templates'] = templates
# Add to list and pickle
(add_status, add_msg) = vgrid_add_triggers(configuration, vgrid_name,
[rule_dict], update_id, rank)
if not add_status:
logger.error('%s failed to add/update trigger: %s' %
(client_id, add_msg))
output_objects.append({
'object_type': 'error_text',
'text': '%s' % add_msg
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if rank is not None:
logger.info('%s moved trigger %s to %d' % (client_id, rule_id, rank))
output_objects.append({
'object_type':
'text',
'text':
'moved %s trigger %s to position %d' % (vgrid_name, rule_id, rank)
})
elif update_id:
logger.info('%s updated trigger: %s' % (client_id, rule_dict))
output_objects.append({
'object_type':
'text',
'text':
'Existing trigger %s successfully updated in %s %s!' %
(rule_id, vgrid_name, label)
})
else:
logger.info('%s added new trigger: %s' % (client_id, rule_dict))
output_objects.append({
'object_type':
'text',
'text':
'New trigger %s successfully added to %s %s!' %
(rule_id, vgrid_name, label)
})
output_objects.append({
'object_type': 'link',
'destination': 'vgridworkflows.py?vgrid_name=%s' % vgrid_name,
'text': 'Back to workflows for %s' % vgrid_name
})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path can use wildcards, dst and current_dir cannot
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
dst = accepted['dst'][-1]
current_dir = accepted['current_dir'][-1].lstrip(os.sep)
# All paths are relative to current_dir
pattern_list = [os.path.join(current_dir, i) for i in pattern_list]
if dst:
dst = os.path.join(current_dir, dst)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dir = os.path.abspath(
os.path.join(base_dir, current_dir.lstrip(os.sep)))
if not valid_user_path(configuration, abs_dir, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"You're not allowed to work in %s!" % current_dir
})
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_dir, current_dir))
return (output_objects, returnvalues.CLIENT_ERROR)
if verbose(flags):
output_objects.append({
'object_type': 'text',
'text': "working in %s" % current_dir
})
if dst:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# NOTE: dst already incorporates current_dir prefix here
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
logger.info('chksum in %s' % abs_dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % dst
})
logger.warning('%s tried to %s restricted path %s !(%s)' %
(client_id, op_name, abs_dest, dst))
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_dest))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot checksum to "%s": inside a read-only location!' %
relative_dest
})
return (output_objects, returnvalues.CLIENT_ERROR)
all_lines = []
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
# NOTE: we produce output matching an invocation of:
# du -aL --apparent-size --block-size=1 PATH [PATH ...]
filedus = []
summarize_output = summarize(flags)
for abs_path in match:
if invisible_path(abs_path):
continue
relative_path = abs_path.replace(base_dir, '')
# cache accumulated sub dir sizes - du sums into parent dir size
dir_sizes = {}
try:
# Assume a directory to walk
for (root, dirs, files) in walk(abs_path,
topdown=False,
followlinks=True):
if invisible_path(root):
continue
dir_bytes = 0
for name in files:
real_file = os.path.join(root, name)
if invisible_path(real_file):
continue
relative_file = real_file.replace(base_dir, '')
size = os.path.getsize(real_file)
dir_bytes += size
if not summarize_output:
filedus.append({
'object_type': 'filedu',
'name': relative_file,
'bytes': size
})
for name in dirs:
real_dir = os.path.join(root, name)
if invisible_path(real_dir):
continue
dir_bytes += dir_sizes[real_dir]
relative_root = root.replace(base_dir, '')
dir_bytes += os.path.getsize(root)
dir_sizes[root] = dir_bytes
if root == abs_path or not summarize_output:
filedus.append({
'object_type': 'filedu',
'name': relative_root,
'bytes': dir_bytes
})
if os.path.isfile(abs_path):
# Fall back to plain file where walk is empty
size = os.path.getsize(abs_path)
filedus.append({
'object_type': 'filedu',
'name': relative_path,
'bytes': size
})
except Exception as exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if dst:
all_lines += [
'%(bytes)d\t\t%(name)s\n' % entry for entry in filedus
]
else:
output_objects.append({
'object_type': 'filedus',
'filedus': filedus
})
if dst and not write_file(''.join(all_lines), abs_dest, logger):
output_objects.append({
'object_type':
'error_text',
'text':
"failed to write disk use to %s" % relative_dest
})
logger.error("writing disk use to %s for %s failed" %
(abs_dest, client_id))
status = returnvalues.SYSTEM_ERROR
return (output_objects, status)
def run_transfer(configuration, client_id, transfer_dict):
"""Actual data transfer built from transfer_dict on behalf of client_id"""
logger.debug('run transfer for %s: %s' %
(client_id, blind_pw(transfer_dict)))
transfer_id = transfer_dict['transfer_id']
action = transfer_dict['action']
protocol = transfer_dict['protocol']
status_dir = get_status_dir(configuration, client_id, transfer_id)
cmd_map = get_cmd_map()
if not protocol in cmd_map[action]:
raise ValueError('unsupported protocol: %s' % protocol)
client_dir = client_id_dir(client_id)
makedirs_rec(status_dir, configuration)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
# TODO: we should refactor to move command extraction into one function
command_pattern = cmd_map[action][protocol]
target_helper_list = []
key_path = transfer_dict.get("key", "")
if key_path:
# Use key with given name from settings dir
settings_base_dir = os.path.abspath(
os.path.join(configuration.user_settings, client_dir)) + os.sep
key_path = os.path.join(settings_base_dir, user_keys_dir,
key_path.lstrip(os.sep))
# IMPORTANT: path must be expanded to abs for proper chrooting
key_path = os.path.abspath(key_path)
if not valid_user_path(configuration, key_path, settings_base_dir):
logger.error('rejecting illegal directory traversal for %s (%s)' %
(key_path, blind_pw(transfer_dict)))
raise ValueError("user provided a key outside own settings!")
rel_src_list = transfer_dict['src']
rel_dst = transfer_dict['dst']
compress = transfer_dict.get("compress", False)
exclude = transfer_dict.get("exclude", [])
if transfer_dict['action'] in ('import', ):
logger.debug('setting abs dst for action %(action)s' % transfer_dict)
src_path_list = transfer_dict['src']
dst_path = os.path.join(base_dir, rel_dst.lstrip(os.sep))
dst_path = os.path.abspath(dst_path)
for src in rel_src_list:
abs_dst = os.path.join(dst_path, src.lstrip(os.sep))
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dst = os.path.abspath(abs_dst)
# Reject illegal directory traversal and hidden files
if not valid_user_path(configuration, abs_dst, base_dir, True):
logger.error(
'rejecting illegal directory traversal for %s (%s)' %
(abs_dst, blind_pw(transfer_dict)))
raise ValueError("user provided a destination outside home!")
if src.endswith(os.sep):
target_helper_list.append(
(get_lftp_target(True, False, exclude),
get_rsync_target(True, False, exclude, compress)))
else:
target_helper_list.append(
(get_lftp_target(True, True, exclude),
get_rsync_target(True, True, exclude, compress)))
makedirs_rec(dst_path, configuration)
elif transfer_dict['action'] in ('export', ):
logger.debug('setting abs src for action %(action)s' % transfer_dict)
dst_path = transfer_dict['dst']
src_path_list = []
for src in rel_src_list:
src_path = os.path.join(base_dir, src.lstrip(os.sep))
# IMPORTANT: path must be expanded to abs for proper chrooting
src_path = os.path.abspath(src_path)
# Reject illegal directory traversal and hidden files
if not valid_user_path(configuration, src_path, base_dir, True):
logger.error(
'rejecting illegal directory traversal for %s (%s)' %
(src, blind_pw(transfer_dict)))
raise ValueError("user provided a source outside home!")
src_path_list.append(src_path)
if src.endswith(os.sep) or os.path.isdir(src):
target_helper_list.append(
(get_lftp_target(False, False, exclude),
get_rsync_target(False, False, exclude, compress)))
else:
target_helper_list.append(
(get_lftp_target(False, True, exclude),
get_rsync_target(False, True, exclude, compress)))
else:
raise ValueError('unsupported action for %(transfer_id)s: %(action)s' %
transfer_dict)
run_dict = transfer_dict.copy()
run_dict['log_path'] = os.path.join(status_dir, 'transfer.log')
# Use private known hosts file for ssh transfers as explained above
# NOTE: known_hosts containing '=' silently leads to rest getting ignored!
# use /dev/null to skip host key verification completely for now.
#run_dict['known_hosts'] = os.path.join(base_dir, '.ssh', 'known_hosts')
run_dict['known_hosts'] = '/dev/null'
# Make sure password is set to empty string as default
run_dict['password'] = run_dict.get('password', '')
# TODO: this is a bogus cert path for now - we don't support ssl certs
run_dict['cert'] = run_dict.get('cert', '')
# IMPORTANT: must be implicit proto or 'ftp://' (not ftps://) and similarly
# webdav(s) must use explicit http(s) instead. In both cases we
# replace protocol between cmd selection and lftp path expansion
if run_dict['protocol'] == 'ftps':
run_dict['orig_proto'] = run_dict['protocol']
run_dict['protocol'] = 'ftp'
logger.info(
'force %(orig_proto)s to %(protocol)s for %(transfer_id)s' %
run_dict)
elif run_dict['protocol'].startswith('webdav'):
run_dict['orig_proto'] = run_dict['protocol']
run_dict['protocol'] = run_dict['protocol'].replace('webdav', 'http')
logger.info(
'force %(orig_proto)s to %(protocol)s for %(transfer_id)s' %
run_dict)
if key_path:
rel_key = run_dict['key']
rel_cert = run_dict['cert']
run_dict['key'] = key_path
run_dict['cert'] = key_path.replace(rel_key, rel_cert)
run_dict['ssh_auth'] = get_ssh_auth(True, run_dict)
run_dict['ssl_auth'] = get_ssl_auth(True, run_dict)
else:
# Extract encrypted password
password_digest = run_dict.get('password_digest', '')
if password_digest:
_, _, _, payload = password_digest.split("$")
unscrambled = unscramble_digest(configuration.site_digest_salt,
payload)
_, _, password = unscrambled.split(":")
run_dict['password'] = password
run_dict['ssh_auth'] = get_ssh_auth(False, run_dict)
run_dict['ssl_auth'] = get_ssl_auth(False, run_dict)
run_dict['rel_dst'] = rel_dst
run_dict['dst'] = dst_path
run_dict['lftp_buf_size'] = run_dict.get('lftp_buf_size',
lftp_buffer_bytes)
run_dict['lftp_sftp_block_size'] = run_dict.get('sftp_sftp_block_size',
lftp_sftp_block_bytes)
status = 0
for (src, rel_src, target_helper) in zip(src_path_list, rel_src_list,
target_helper_list):
(lftp_target, rsync_target) = target_helper
logger.debug('setting up %(action)s for %(src)s' % run_dict)
if run_dict['protocol'] == 'sftp' and not os.path.isabs(src):
# NOTE: lftp interprets sftp://FQDN/SRC as absolute path /SRC
# We force relative paths into user home with a tilde.
# The resulting sftp://FQDN/~/SRC looks funky but works.
run_dict['src'] = "~/%s" % src
else:
# All other paths are probably absolute or auto-chrooted anyway
run_dict['src'] = src
run_dict['rel_src'] = rel_src
run_dict['lftp_args'] = ' '.join(lftp_target[0]) % run_dict
run_dict['lftp_excludes'] = ' '.join(lftp_target[1])
# src and dst may actually be reversed for lftp, but for symmetry ...
run_dict['lftp_src'] = lftp_target[2][0] % run_dict
run_dict['lftp_dst'] = lftp_target[2][1] % run_dict
run_dict['rsync_args'] = ' '.join(rsync_target[0]) % run_dict
# Preserve excludes on list form for rsync, where it matters
run_dict[RSYNC_EXCLUDES_LIST] = rsync_target[1]
run_dict['rsync_src'] = rsync_target[2][0] % run_dict
run_dict['rsync_dst'] = rsync_target[2][1] % run_dict
blind_dict = blind_pw(run_dict)
logger.debug('expanded vars to %s' % blind_dict)
# NOTE: Make sure NOT to break rsync excludes on list form as they
# won't work if concatenated to a single string in command_list!
command_list, blind_list = [], []
for i in command_pattern:
if i == RSYNC_EXCLUDES_LIST:
command_list += run_dict[RSYNC_EXCLUDES_LIST]
blind_list += run_dict[RSYNC_EXCLUDES_LIST]
else:
command_list.append(i % run_dict)
blind_list.append(i % blind_dict)
command_str = ' '.join(command_list)
blind_str = ' '.join(blind_list)
logger.info('run %s on behalf of %s' % (blind_str, client_id))
transfer_proc = subprocess_popen(command_list,
stdout=subprocess_pipe,
stderr=subprocess_pipe)
# Save transfer_proc.pid for use in clean up during shutdown
# in that way we can resume pretty smoothly in next run.
sub_pid = transfer_proc.pid
logger.info('%s %s running transfer process %s' %
(client_id, transfer_id, sub_pid))
add_sub_pid(configuration, sub_pid_map, client_id, transfer_id,
sub_pid)
out, err = transfer_proc.communicate()
exit_code = transfer_proc.wait()
status |= exit_code
del_sub_pid(configuration, sub_pid_map, client_id, transfer_id,
sub_pid)
logger.info('done running transfer %s: %s' % (transfer_id, blind_str))
logger.debug('raw output is: %s' % out)
logger.debug('raw error is: %s' % err)
logger.debug('result was %s' % exit_code)
if not transfer_result(configuration, client_id, run_dict, exit_code,
out.replace(base_dir, ''),
err.replace(base_dir, '')):
logger.error('writing transfer status for %s failed' % transfer_id)
logger.debug('done handling transfers in %(transfer_id)s' % transfer_dict)
transfer_dict['exit_code'] = status
if status == 0:
transfer_dict['status'] = 'DONE'
else:
transfer_dict['status'] = 'FAILED'
